CMakeList is a minimal:
project(custom_op)

find_package(Torch REQUIRED)

add_executable(jit_minimal jit_minimal.cpp)
target_link_libraries(jit_minimal "${TORCH_LIBRARIES}")
set_property(TARGET jit_minimal PROPERTY CXX_STANDARD 11)

Command used for build -
mkdir -p build/
cd build/
# cmake -DCMAKE_PREFIX_PATH=~/libs/libtorch/ ..
cmake -DCMAKE_PREFIX_PATH=~/libs/nightly/libtorch/ ..
make

And I got the libtorch using the download url on pytorch website.