Sir I tried both…
CMakeLists.txt File
cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
set(CMAKE_VERBOSE_MAKEFILE ON)
set(PUBLIC_HEADERS
    pytorch-rnnlm.h
)

add_library(kaldi-pyrnnlm
    pytorch-rnnlm.cc
)

#set(D_GLIBCXX_USE_CXX11_ABI,"0")
list(APPEND CMAKE_PREFIX_PATH "/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch")


target_include_directories(kaldi-pyrnnlm PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/torch/csrc/api/include>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/caffe2>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/ATen/core>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/c10/util>
#PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib>
#PUBLIC $<BUILD_INTERFACE:/usr/local/lib/python3.6/dist-packages/torch/lib>
#PUBLIC $<BUILD_INTERFACE:/usr/local/cuda/lib64>
#PUBLIC $<BUILD_INTERFACE:/usr/local/cuda/lib64/stubs>
)

#link_directories(/home/rakesh/rishabh_workspace/ASR/kaldi_old/tools/libtorch)

#target_include_directories(kaldi-pyrnnlm PUBLIC 
#    ${PYTORCH_DIR}
#    ${PYTORCH_DIR}/bin
#    ${PYTORCH_DIR}/include
#    ${PYTORCH_DIR}/share
#
#)

#set(CMAKE_CXX_STANDARD 14)

find_package(Torch REQUIRED PATHS /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch)

target_include_directories(kaldi-pyrnnlm PUBLIC 
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>
     $<INSTALL_INTERFACE:include/kaldi>
     #/home/rakesh/rishabh_workspace/ASR/kaldi_old/tools/libtorch/include
     #/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/torch/csrc/api/include
)

target_compile_options(kaldi-pyrnnlm PUBLIC 
    -lz
    -ldl
    -fPIC
)

target_link_libraries(kaldi-pyrnnlm PUBLIC
    kaldi-lm
    kaldi-util
    kaldi-matrix
    kaldi-base
    #libc10.so
    #libc10_cuda.so
    #-lz
    #-ldl
    #-fPIC
    #${TORCH_LIBRARIES}
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libtorch.so
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libc10.so
    /usr/local/cuda/lib64/libnvrtc.so
    /usr/local/cuda/lib64/stubs/libcuda.so;
    /usr/local/cuda/lib64/libnvToolsExt.so
    /usr/local/cuda/lib64/libcudart.so
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libc10_cuda.so
    #/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libtorch_cpu.so
    #-L/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/cuda/lib64
    #-L/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/cuda/lib64/studs/
)



#find_package(Torch REQUIRED PATHS /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch)

MESSAGE("TORCH LIBS ${TORCH_LIBRARIES}")

#target_link_libraries(kaldi-pyrnnlm "${TORCH_LIBRARIES}")
#set_property(TARGET kaldi-pyrnnlm PROPERTY CXX_STANDARD 14)

install(TARGETS kaldi-pyrnnlm
    EXPORT kaldi-targets
    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
)

install(FILES ${PUBLIC_HEADERS} DESTINATION include/kaldi/pyrnnlm)


First
list(APPEND CMAKE_PREFIX_PATH "/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch")

Uncomment ${TORCH_LIBRARIES} but got the same error
CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch_cpu,INTERFACE_SYSTEM_INCLUDE_DIRECTORIES>

  Target "torch_cpu" not found.


CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch_cpu,INTERFACE_INCLUDE_DIRECTORIES>

  Target "torch_cpu" not found.


CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch_cuda,INTERFACE_SYSTEM_INCLUDE_DIRECTORIES>

  Target "torch_cuda" not found.


CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch_cuda,INTERFACE_INCLUDE_DIRECTORIES>

  Target "torch_cuda" not found.


CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch,INTERFACE_SYSTEM_INCLUDE_DIRECTORIES>

  Target "torch" not found.


CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch,INTERFACE_INCLUDE_DIRECTORIES>

  Target "torch" not found.

So, I went for adding individual library in target_link_libraries
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libtorch.so
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libc10.so
    /usr/local/cuda/lib64/libnvrtc.so
    /usr/local/cuda/lib64/stubs/libcuda.so;
    /usr/local/cuda/lib64/libnvToolsExt.so
    /usr/local/cuda/lib64/libcudart.so
    /home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libc10_cuda.so

And comment ${TORCH_LIBRARIES} but got the error
/usr/bin/ld: ../pyrnnlm/libkaldi-pyrnnlm.a(pytorch-rnnlm.cc.o): undefined reference to symbol '_ZN3c107IntType3getEv'
//home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libtorch_cpu.so: error adding symbols: DSO missing from command line
collect2: error: ld returned 1 exit status
src/pyrnnlmbin/CMakeFiles/lattice-lmrescore-py-rnnlm.dir/build.make:128: recipe for target 'src/pyrnnlmbin/lattice-lmrescore-py-rnnlm' failed

And when I add this also in the target_link_libraries
/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib/libtorch_cpu.so

Got the main errors
lattice-lmrescore-py-rnnlm.cc:(.text._ZN3c1019UndefinedTensorImpl9singletonEv[_ZN3c1019UndefinedTensorImpl9singletonEv]+0x7): undefined reference to `c10::UndefinedTensorImpl::_singleton'
CMakeFiles/lattice-lmrescore-py-rnnlm.dir/lattice-lmrescore-py-rnnlm.cc.o: In function `c10::IValue::toDouble() const':
lattice-lmrescore-py-rnnlm.cc:(.text._ZNK3c106IValue8toDoubleEv[_ZNK3c106IValue8toDoubleEv]+0xa0): undefined reference to `c10::Error::Error(c10::SourceLocation, std::string const&)'
CMakeFiles/lattice-lmrescore-py-rnnlm.dir/lattice-lmrescore-py-rnnlm.cc.o: In function `c10::IValue::toInt() const':
lattice-lmrescore-py-rnnlm.cc:(.text._ZNK3c106IValue5toIntEv[_ZNK3c106IValue5toIntEv]+0xa0): undefined reference to `c10::Error::Error(c10::SourceLocation, std::string const&)'
CMakeFiles/lattice-lmrescore-py-rnnlm.dir/lattice-lmrescore-py-rnnlm.cc.o: In function `c10::IValue::toTensor() const &':
...
...
_ZTIN5torch8autograd12AutogradMetaE]+0x10): undefined reference to `typeinfo for c10::AutogradMetaInterface'
collect2: error: ld returned 1 exit status
src/pyrnnlmbin/CMakeFiles/lattice-lmrescore-py-rnnlm.dir/build.make:122: recipe for target 'src/pyrnnlmbin/lattice-lmrescore-py-rnnlm' failed

Repository : https://github.com/cyfer0618/kaldi-pytorch-rnnlm/blob/master/kaldi/src/pyrnnlm/CMakeLists.txt 1