Here is the trace for the threads:
(gdb) thread apply all bt

Thread 4 (Thread 0x7facf308c700 (LWP 16191)):
#0  0x00007fad7454bd1f in accept4 () from /lib64/libc.so.6
#1  0x00007facf32b6a5a in ?? () from /lib64/libcuda.so.1
#2  0x00007facf32a85bd in ?? () from /lib64/libcuda.so.1
#3  0x00007facf32b8118 in ?? () from /lib64/libcuda.so.1
#4  0x00007fad74f2ae65 in start_thread () from /lib64/libpthread.so.0
#5  0x00007fad7454a88d in clone () from /lib64/libc.so.6


Thread 3 (Thread 0x7facf288b700 (LWP 16192)):
#0  0x00007fad74f2e9f5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fad644e582c in std::condition_variable::wait(std::unique_lockstd::mutex&) ()
from /lib64/libstdc++.so.6
#2  0x00007fad1dc8ce23 in torch::autograd::ReadyQueue::pop() ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch.so
#3  0x00007fad1dc8ed7c in torch::autograd::Engine::thread_main(std::shared_ptr<torch::autogr                                                     ad::GraphTask> const&, bool) ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch.so
#4  0x00007fad1dc88979 in torch::autograd::Engine::thread_init(int) ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch.so
#5  0x00007fad64d9408a in torch::autograd::python::PythonEngine::thread_init(int) ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch_python.so
#6  0x00007fad659afdef in execute_native_thread_routine ()
—Type  to continue, or q  to quit—return
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/_C.cpython-36m-x86_64-li                                                     nux-gnu.so
#7  0x00007fad74f2ae65 in start_thread () from /lib64/libpthread.so.0
#8  0x00007fad7454a88d in clone () from /lib64/libc.so.6


Thread 2 (Thread 0x7facf208a700 (LWP 16193)):
#0  0x00007fad74f2e9f5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007fad644e582c in std::condition_variable::wait(std::unique_lockstd::mutex&) ()
from /lib64/libstdc++.so.6
#2  0x00007fad1dc8ce23 in torch::autograd::ReadyQueue::pop() ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch.so
#3  0x00007fad1dc8ed7c in torch::autograd::Engine::thread_main(std::shared_ptr<torch::autogr                                                     ad::GraphTask> const&, bool) ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch.so
#4  0x00007fad1dc88979 in torch::autograd::Engine::thread_init(int) ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch.so
#5  0x00007fad64d9408a in torch::autograd::python::PythonEngine::thread_init(int) ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch_python.so
#6  0x00007fad659afdef in execute_native_thread_routine ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/_C.cpython-36m-x86_64-li                                                     nux-gnu.so
#7  0x00007fad74f2ae65 in start_thread () from /lib64/libpthread.so.0
#8  0x00007fad7454a88d in clone () from /lib64/libc.so.6
—Type  to continue, or q  to quit—return


Thread 1 (Thread 0x7fad75853740 (LWP 16174)):
#0  0x00007fad744d2424 in memalign () from /lib64/libc.so.6
#1  0x00007fad744d404c in posix_memalign () from /lib64/libc.so.6
#2  0x00007fad1960ab4a in c10::alloc_cpu(unsigned long) ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libc10.so
#3  0x00007fad1960c5fa in c10::DefaultCPUAllocator::allocate(unsigned long) const ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libc10.so
#4  0x00007fad1b93c78a in at::native::empty_cpu(c10::ArrayRef, c10::TensorOptions cons                                                     t&, c10::optionalc10::MemoryFormat) ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch.so
#5  0x00007fad1bb8530b in at::CPUType::(anonymous namespace)::empty(c10::ArrayRef, c10                                                     ::TensorOptions const&, c10::optionalc10::MemoryFormat) ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch.so
#6  0x00007fad1bbccb37 in c10::detail::wrap_kernel_functor_unboxed_<c10::detail::WrapRuntime                                                     KernelFunctor_<at::Tensor ()(c10::ArrayRef, c10::TensorOptions const&, c10::optional<                                                     c10::MemoryFormat>), at::Tensor, c10::guts::typelist::typelist<c10::ArrayRef, c10::Ten                                                     sorOptions const&, c10::optionalc10::MemoryFormat > >, at::Tensor (c10::ArrayRef, c1                                                     0::TensorOptions const&, c10::optionalc10::MemoryFormat)>::call(c10::OperatorKernel, c10:                                                     :ArrayRef, c10::TensorOptions const&, c10::optionalc10::MemoryFormat) ()
from /home/nfs/USER/.local/lib/python3.6/site-packages/torch/lib/libtorch.so
#7  0x00007fad1db8c795 in torch::autograd::VariableType::(anonymous namespace)::empty(c10::A

It does seem like it is autograd that is using extra threads, despite the fact that there is no GPU to make use of.