I’ve tried few other things and below change on caffe2/CMakeList.txt worked!
diff --git a/caffe2/CMakeLists.txt b/caffe2/CMakeLists.txt
index 8025a7de3c..8e94978e72 100644
--- a/caffe2/CMakeLists.txt
+++ b/caffe2/CMakeLists.txt
@@ -561,7 +561,7 @@ if (NOT INTERN_BUILD_MOBILE OR NOT BUILD_CAFFE2_MOBILE)
       ${TORCH_SRC_DIR}/csrc/cuda/comm.cpp
       ${TORCH_SRC_DIR}/csrc/jit/tensorexpr/cuda_codegen.cpp
     )
-    add_library(caffe2_nvrtc SHARED ${ATen_NVRTC_STUB_SRCS})
+    add_library(caffe2_nvrtc ${ATen_NVRTC_STUB_SRCS})
     target_link_libraries(caffe2_nvrtc ${CUDA_NVRTC} ${CUDA_CUDA_LIB} ${CUDA_NVRTC_LIB})
     target_include_directories(caffe2_nvrtc PRIVATE ${CUDA_INCLUDE_DIRS})
     install(TARGETS caffe2_nvrtc DESTINATION "${TORCH_INSTALL_LIB_DIR}")
@@ -703,6 +703,9 @@ ELSEIF(USE_CUDA)
   cuda_add_library(torch_cuda ${Caffe2_GPU_SRCS})
   set(CUDA_LINK_LIBRARIES_KEYWORD)
   torch_compile_options(torch_cuda)  # see cmake/public/utils.cmake
+  if (NOT BUILD_SHARED_LIBS)
+    target_compile_definitions(torch_cuda PRIVATE USE_DIRECT_NVRTC)
+  endif()
   if (USE_NCCL)
     target_link_libraries(torch_cuda PRIVATE __caffe2_nccl)

In aten/src/ATen/cuda/detail/CUDAHooks.cpp, “#ifdef USE_DIRECT_NVRTC” directive is used.
But “USE_DIRECT_NVRTC” was not defined in any CMakeList.txt and because of that,
application linked with satic libtorch tries to load “caffe2_nvrtc.dll”.