Sir, I try the first three points. I did make a small example which was working fine with the same way I wrote this code. When I run the code pyrnnlm/CMakeLists.txt (with kaldi and without Kaldi) independently then it works fine. The pyrnnlmbin/CMakeLists.txt is calling pyrnnlm/CMakeLists.txt along with some kaldi CMakeLists.txt then there is an error.
a. I tried CMAKE_PREFIX_PATH but when the pyrnnlmbin/CMakeLists.txt is calling pyrnnlm/CMakeLists.txt then it is not working which is why I have to add each and every library separately.
b. I did this now. But then also I am getting the error.
c. you have libc10_cuda.so but u don’t have libtorch_gpu.so . I have added all the library which u have mentioned above. But getting the same error as before.  I did all three points a, b and c points together.
Error
CMakeFiles/lattice-lmrescore-py-rnnlm.dir/lattice-lmrescore-py-rnnlm.cc.o: In function `c10::IValue::toDouble() const':
lattice-lmrescore-py-rnnlm.cc:(.text._ZNK3c106IValue8toDoubleEv[_ZNK3c106IValue8toDoubleEv]+0xa0): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
CMakeFiles/lattice-lmrescore-py-rnnlm.dir/lattice-lmrescore-py-rnnlm.cc.o: In function `c10::IValue::toInt() const':
lattice-lmrescore-py-rnnlm.cc:(.text._ZNK3c106IValue5toIntEv[_ZNK3c106IValue5toIntEv]+0xa0): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
CMakeFiles/lattice-lmrescore-py-rnnlm.dir/lattice-lmrescore-py-rnnlm.cc.o: In function `c10::IValue::toTensor() const &':
lattice-lmrescore-py-rnnlm.cc:(.text._ZNKR3c106IValue8toTensorEv[_ZNKR3c106IValue8toTensorEv]+0xeb): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
CMakeFiles/lattice-lmrescore-py-rnnlm.dir/lattice-lmrescore-py-rnnlm.cc.o: In function `c10::intrusive_ptr<c10::TensorImpl, c10::UndefinedTensorImpl>::retain_()':
lattice-lmrescore-py-rnnlm.cc:(.text._ZN3c1013intrusive_ptrINS_10TensorImplENS_19UndefinedTensorImplEE7retain_Ev[_ZN3c1013intrusive_ptrINS_10TensorImplENS_19UndefinedTensorImplEE7retain_Ev]+0xe1): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `c10::scalarTypeToTypeMeta(c10::ScalarType)':
pytorch-rnnlm.cc:(.text+0x307): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o):pytorch-rnnlm.cc:(.text+0xa04): more undefined references to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)' follow
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `kaldi::py_rnnlm::KaldiPyRnnlmWrapper::ReadPyModel(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int)':
pytorch-rnnlm.cc:(.text+0x1319): undefined reference to `torch::jit::load(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, c10::optional<c10::Device>, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > >&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `kaldi::py_rnnlm::KaldiPyRnnlmWrapper::AcquireInitialTensors()':
pytorch-rnnlm.cc:(.text+0x210f): undefined reference to `torch::jit::Method::operator()(std::vector<c10::IValue, std::allocator<c10::IValue> >, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, c10::IValue, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, c10::IValue> > > const&)'
pytorch-rnnlm.cc:(.text+0x245d): undefined reference to `torch::jit::Method::operator()(std::vector<c10::IValue, std::allocator<c10::IValue> >, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, c10::IValue, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, c10::IValue> > > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `kaldi::py_rnnlm::KaldiPyRnnlmWrapper::GetLogProb(int, int, at::Tensor const&, at::Tensor const&, at::Tensor*, at::Tensor*)':
pytorch-rnnlm.cc:(.text+0x2a79): undefined reference to `torch::jit::Method::operator()(std::vector<c10::IValue, std::allocator<c10::IValue> >, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, c10::IValue, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, c10::IValue> > > const&)'
pytorch-rnnlm.cc:(.text+0x2cba): undefined reference to `torch::jit::Method::operator()(std::vector<c10::IValue, std::allocator<c10::IValue> >, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, c10::IValue, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, c10::IValue> > > const&)'
pytorch-rnnlm.cc:(.text+0x2f2c): undefined reference to `torch::jit::Method::operator()(std::vector<c10::IValue, std::allocator<c10::IValue> >, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, c10::IValue, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, c10::IValue> > > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o):pytorch-rnnlm.cc:(.text+0x319e): more undefined references to `torch::jit::Method::operator()(std::vector<c10::IValue, std::allocator<c10::IValue> >, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, c10::IValue, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, c10::IValue> > > const&)' follow
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `c10::Device::validate()':
pytorch-rnnlm.cc:(.text._ZN3c106Device8validateEv[_ZN3c106Device8validateEv]+0x10f): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
pytorch-rnnlm.cc:(.text._ZN3c106Device8validateEv[_ZN3c106Device8validateEv]+0x25c): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `c10::intrusive_ptr_target::~intrusive_ptr_target()':
pytorch-rnnlm.cc:(.text._ZN3c1020intrusive_ptr_targetD2Ev[_ZN3c1020intrusive_ptr_targetD5Ev]+0xf8): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
pytorch-rnnlm.cc:(.text._ZN3c1020intrusive_ptr_targetD2Ev[_ZN3c1020intrusive_ptr_targetD5Ev]+0x1f5): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `c10::IValue::toTensor() &&':
pytorch-rnnlm.cc:(.text._ZNO3c106IValue8toTensorEv[_ZNO3c106IValue8toTensorEv]+0xeb): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o):pytorch-rnnlm.cc:(.text._ZNKR3c106IValue9toIntListEv[_ZNKR3c106IValue9toIntListEv]+0xeb): more undefined references to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)' follow
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `torch::jit::Object::get_method(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) const':
pytorch-rnnlm.cc:(.text._ZNK5torch3jit6Object10get_methodERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE[_ZNK5torch3jit6Object10get_methodERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE]+0x4e): undefined reference to `torch::jit::Object::find_method(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) const'
pytorch-rnnlm.cc:(.text._ZNK5torch3jit6Object10get_methodERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE[_ZNK5torch3jit6Object10get_methodERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE]+0x181): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `c10::intrusive_ptr<c10::ivalue::Object, c10::detail::intrusive_target_default_null_type<c10::ivalue::Object> >::retain_()':
pytorch-rnnlm.cc:(.text._ZN3c1013intrusive_ptrINS_6ivalue6ObjectENS_6detail34intrusive_target_default_null_typeIS2_EEE7retain_Ev[_ZN3c1013intrusive_ptrINS_6ivalue6ObjectENS_6detail34intrusive_target_default_null_typeIS2_EEE7retain_Ev]+0xe1): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `c10::intrusive_ptr<c10::detail::ListImpl, c10::detail::intrusive_target_default_null_type<c10::detail::ListImpl> >::retain_()':
pytorch-rnnlm.cc:(.text._ZN3c1013intrusive_ptrINS_6detail8ListImplENS1_34intrusive_target_default_null_typeIS2_EEE7retain_Ev[_ZN3c1013intrusive_ptrINS_6detail8ListImplENS1_34intrusive_target_default_null_typeIS2_EEE7retain_Ev]+0xe1): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `at::Tensor& c10::KernelFunction::callUnboxed<at::Tensor&, at::Tensor&, c10::Scalar>(c10::OperatorHandle const&, at::Tensor&, c10::Scalar) const':
pytorch-rnnlm.cc:(.text._ZNK3c1014KernelFunction11callUnboxedIRN2at6TensorEJS4_NS_6ScalarEEEET_RKNS_14OperatorHandleEDpT0_[_ZNK3c1014KernelFunction11callUnboxedIRN2at6TensorEJS4_NS_6ScalarEEEET_RKNS_14OperatorHandleEDpT0_]+0x13a): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): In function `at::Tensor c10::KernelFunction::callUnboxed<at::Tensor, at::Tensor const&, long, long>(c10::OperatorHandle const&, at::Tensor const&, long, long) const':
pytorch-rnnlm.cc:(.text._ZNK3c1014KernelFunction11callUnboxedIN2at6TensorEJRKS3_llEEET_RKNS_14OperatorHandleEDpT0_[_ZNK3c1014KernelFunction11callUnboxedIN2at6TensorEJRKS3_llEEET_RKNS_14OperatorHandleEDpT0_]+0x16f): undefined reference to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)'
../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o):pytorch-rnnlm.cc:(.text._ZNK3c1014KernelFunction11callUnboxedIN2at6TensorEJRKS3_RKNS_13TensorOptionsEbbNS_8optionalINS_12MemoryFormatEEEEEET_RKNS_14OperatorHandleEDpT0_[_ZNK3c1014KernelFunction11callUnboxedIN2at6TensorEJRKS3_RKNS_13TensorOptionsEbbNS_8optionalINS_12MemoryFormatEEEEEET_RKNS_14OperatorHandleEDpT0_]+0x1db): more undefined references to `c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)' follow
collect2: error: ld returned 1 exit status
src/pyrnnlmbin/CMakeFiles/lattice-lmrescore-py-rnnlm.dir/build.make:131: recipe for target 'src/pyrnnlmbin/lattice-lmrescore-py-rnnlm' failed
make[2]: *** [src/pyrnnlmbin/lattice-lmrescore-py-rnnlm] Error 1
CMakeFiles/Makefile2:7492: recipe for target 'src/pyrnnlmbin/CMakeFiles/lattice-lmrescore-py-rnnlm.dir/all' failed
make[1]: *** [src/pyrnnlmbin/CMakeFiles/lattice-lmrescore-py-rnnlm.dir/all] Error 2

pyrnnlm/CMakeLists.txt
cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
set(CMAKE_VERBOSE_MAKEFILE ON)
set(PUBLIC_HEADERS
    pytorch-rnnlm.h
)

add_library(kaldi-pyrnnlm
    pytorch-rnnlm.cc
)

#set(D_GLIBCXX_USE_CXX11_ABI,"0")
#list(APPEND CMAKE_PREFIX_PATH "/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch")


target_include_directories(kaldi-pyrnnlm PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/torch/csrc/api/include>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/caffe2>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/ATen/core>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/c10/util>
#PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib>
#PUBLIC $<BUILD_INTERFACE:/usr/local/lib/python3.6/dist-packages/torch/lib>
#PUBLIC $<BUILD_INTERFACE:/usr/local/cuda/lib64>
#PUBLIC $<BUILD_INTERFACE:/usr/local/cuda/lib64/stubs>
)

#link_directories(/home/rakesh/rishabh_workspace/ASR/kaldi_old/tools/libtorch)

#target_include_directories(kaldi-pyrnnlm PUBLIC 
#    ${PYTORCH_DIR}
#    ${PYTORCH_DIR}/bin
#    ${PYTORCH_DIR}/include
#    ${PYTORCH_DIR}/share
#
#)

#set(CMAKE_CXX_STANDARD 14)

find_package(Torch REQUIRED PATHS /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch)

target_include_directories(kaldi-pyrnnlm PUBLIC 
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>
     $<INSTALL_INTERFACE:include/kaldi>
     #/home/rakesh/rishabh_workspace/ASR/kaldi_old/tools/libtorch/include
     #/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/torch/csrc/api/include
)

target_compile_options(kaldi-pyrnnlm PUBLIC 
    #-lz
    #-ldl
    -fPIC
)

target_link_libraries(kaldi-pyrnnlm PUBLIC
    kaldi-lm
    kaldi-util
    kaldi-matrix
    kaldi-base
    #libc10.so
    #libc10_cuda.so
    -lz
    -ldl
    #-fPIC
    #${TORCH_LIBRARIES}
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libtorch.so
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libc10.so
    /usr/local/cuda/lib64/libnvrtc.so
    /usr/local/cuda/lib64/stubs/libcuda.so;
    /usr/local/cuda/lib64/libnvToolsExt.so
    /usr/local/cuda/lib64/libcudart.so
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libc10_cuda.so
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libtorch_cpu.so
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libtorch_python.so
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libtorch_cuda.so
    #-L/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/cuda/lib64
    #-L/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/cuda/lib64/studs/
)



#find_package(Torch REQUIRED PATHS /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch)

MESSAGE("TORCH LIBS ${TORCH_LIBRARIES}")

#target_link_libraries(kaldi-pyrnnlm "${TORCH_LIBRARIES}")
#set_property(TARGET kaldi-pyrnnlm PROPERTY CXX_STANDARD 14)

install(TARGETS kaldi-pyrnnlm
    EXPORT kaldi-targets
    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
)

install(FILES ${PUBLIC_HEADERS} DESTINATION include/kaldi/pyrnnlm)


pyrnnlmbin/CMakeLists.txt
cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
set(CMAKE_VERBOSE_MAKEFILE ON)
#find_package(Torch REQUIRED)
#find_package(Torch REQUIRED PATHS /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch)

set(CMAKE_CXX_STANDARD 14)

add_kaldi_executable(NAME lattice-lmrescore-py-rnnlm SOURCES lattice-lmrescore-py-rnnlm.cc DEPENDS kaldi-pyrnnlm kaldi-lat)

#target_link_libraries(lattice-lmrescore-py-rnnlm "${PYTORCH_DIR}")
#set_property(TARGET lattice-lmrescore-py-rnnlm PROPERTY CXX_STANDARD 14)


When pyrnnlm/CMakeLists.txt is compiled along with kaldi than there is no error and when the pyrnnlmbin/CMakeLists.txt is calling pyrnnlm/CMakeLists.txt than there is an error.