When I kept as you said

target_link_libraries(kaldi-pyrnnlm PUBLIC
kaldi-lm
kaldi-util
kaldi-matrix
kaldi-base
${TORCH_LIBRARIES}
)

${TORCH_LIBRARIES} in the target_link_libraries gives error as
CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch_cpu,INTERFACE_SYSTEM_INCLUDE_DIRECTORIES>

  Target "torch_cpu" not found.


CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch_cpu,INTERFACE_INCLUDE_DIRECTORIES>

  Target "torch_cpu" not found.


CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch_cuda,INTERFACE_SYSTEM_INCLUDE_DIRECTORIES>

  Target "torch_cuda" not found.


CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch_cuda,INTERFACE_INCLUDE_DIRECTORIES>

  Target "torch_cuda" not found.


CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch,INTERFACE_SYSTEM_INCLUDE_DIRECTORIES>

  Target "torch" not found.


CMake Error:
  Error evaluating generator expression:

    $<TARGET_PROPERTY:torch,INTERFACE_INCLUDE_DIRECTORIES>

  Target "torch" not found.

So, I comment ${TORCH_LIBRARIES}
If I used…
target_include_directories(kaldi-pyrnnlm PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/torch/csrc/api/include>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/caffe2>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/ATen/core>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/include/c10/util>
PUBLIC $<BUILD_INTERFACE:/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib>
PUBLIC $<BUILD_INTERFACE:/usr/local/lib/python3.6/dist-packages/torch/lib>
#PUBLIC $<BUILD_INTERFACE:/usr/local/cuda/lib64>
#PUBLIC $<BUILD_INTERFACE:/usr/local/cuda/lib64/stubs>
)

target_link_libraries(kaldi-pyrnnlm PUBLIC
    kaldi-lm
    kaldi-util
    kaldi-matrix
    kaldi-base
    libc10.so
    libc10_cuda.so
    #-lz
    #-ldl
    #-fPIC
    #${TORCH_LIBRARIES}
    #-L/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch
    #-L/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/libtorch/lib
    #-L/home/rakesh/rishabh_workspace/Garbage/kaldi/tools/cuda/lib64

)


Then I got only 2 error as
/usr/bin/ld: cannot find -lc10
/usr/bin/ld: cannot find -lc10_cuda
collect2: error: ld returned 1 exit status
src/pyrnnlmbin/CMakeFiles/lattice-lmrescore-py-rnnlm.dir/build.make:122: recipe for target 'src/pyrnnlmbin/lattice-lmrescore-py-rnnlm' failed
make[2]: *** [src/pyrnnlmbin/lattice-lmrescore-py-rnnlm] Error 1
CMakeFiles/Makefile2:7492: recipe for target 'src/pyrnnlmbin/CMakeFiles/lattice-lmrescore-py-rnnlm.dir/all' failed
make[1]: *** [src/pyrnnlmbin/CMakeFiles/lattice-lmrescore-py-rnnlm.dir/all] Error 2

Repository : https://github.com/cyfer0618/kaldi-pytorch-rnnlm.git 2