Sorry for the delay, I didn’t have a chance to get to it this weekend. I ran it again and the bt ended again with a call to gettime() but had different upstream methods:
#0  0x00007ffe49b79b44 in clock_gettime ()
#1  0x00007f2a8a03fea6 in __GI___clock_gettime (clock_id=4, tp=0x7ffe49a56670) at ../sysdeps/unix/clock_gettime.c:115
#2  0x00007f281dd2470e in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#3  0x00007f281ddfd837 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#4  0x00007f281de1f7a9 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#5  0x00007f281dcf9d88 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#6  0x00007f281dbfe612 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#7  0x00007f281dc00dcf in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#8  0x00007f281dda4cb5 in cuMemcpyDtoHAsync_v2 () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#9  0x00007f2a8155834f in ?? () from /opt/conda/lib/python3.7/site-packages/torch/lib/libcudart-1581fefa.so.10.0
#10 0x00007f2a81535643 in ?? () from /opt/conda/lib/python3.7/site-packages/torch/lib/libcudart-1581fefa.so.10.0
#11 0x00007f2a81573af8 in cudaMemcpyAsync () from /opt/conda/lib/python3.7/site-packages/torch/lib/libcudart-1581fefa.so.10.0
#12 0x00007f2a1ab508a3 in at::native::_local_scalar_dense_cuda(at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#13 0x00007f2a19a2979b in at::CUDAType::(anonymous namespace)::_local_scalar_dense(at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#14 0x00007f2a173744dd in c10::detail::wrap_kernel_functor_unboxed_<c10::detail::WrapRuntimeKernelFunctor_<c10::Scalar (*)(at::Tensor const&), c10::Scalar, c10::guts::typelist::typelist<at::Tensor const&> >, c10::Scalar (at::Tensor const&)>::call(c10::OperatorKernel*, at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#15 0x00007f2a18f25d6f in torch::autograd::VariableType::(anonymous namespace)::_local_scalar_dense(at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#16 0x00007f2a173744dd in c10::detail::wrap_kernel_functor_unboxed_<c10::detail::WrapRuntimeKernelFunctor_<c10::Scalar (*)(at::Tensor const&), c10::Scalar, c10::guts::typelist::typelist<at::Tensor const&> >, c10::Scalar (at::Tensor const&)>::call(c10::OperatorKernel*, at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#17 0x00007f2a17097dfc in at::_local_scalar_dense(at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#18 0x00007f2a1709895f in at::native::item(at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#19 0x00007f2a174540e0 in at::TypeDefault::item(at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#20 0x00007f2a18fc525b in torch::autograd::VariableType::(anonymous namespace)::item(at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#21 0x00007f2a173744dd in c10::detail::wrap_kernel_functor_unboxed_<c10::detail::WrapRuntimeKernelFunctor_<c10::Scalar (*)(at::Tensor const&), c10::Scalar, c10::guts::typelist::typelist<at::Tensor const&> >, c10::Scalar (at::Tensor const&)>::call(c10::OperatorKernel*, at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#22 0x00007f2a16e085fc in at::Tensor::item() const () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#23 0x00007f2a170d454b in at::native::is_nonzero(at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#24 0x00007f2a19312988 in torch::autograd::VariableType::(anonymous namespace)::is_nonzero(at::Tensor const&) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch.so
#25 0x00007f2a5da57c14 in std::result_of<bool c10::Dispatcher::callUnboxed<bool, at::Tensor const&>(c10::OperatorHandle const&, at::Tensor const&) const::{lambda(c10::DispatchTable const&)#1} (c10::DispatchTable const&)>::type c10::LeftRight<c10::DispatchTable>::read<bool c10::Dispatcher::callUnboxed<bool, at::Tensor const&>(c10::OperatorHandle const&, at::Tensor const&) const::{lambda(c10::DispatchTable const&)#1}>(bool c10::Dispatcher::callUnboxed<bool, at::Tensor const&>(c10::OperatorHandle const&, at::Tensor const&) const::{lambda(c10::DispatchTable const&)#1}&&) const () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_python.so
#26 0x00007f2a5d987d20 in torch::autograd::THPVariable_is_nonzero(_object*, _object*) () from /opt/conda/lib/python3.7/site-packages/torch/lib/libtorch_python.so
#27 0x000055c5e3dbbaba in _PyMethodDef_RawFastCallDict () at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:482
#28 0x000055c5e3dbbc81 in _PyCFunction_FastCallDict (func=0x7f27f816aa50, args=<optimized out>, nargs=<optimized out>, kwargs=<optimized out>) at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:586
#29 0x000055c5e3de8700 in call_unbound_noarg (self=0x7f27f86539b0, func=0x7f27f816aa50, unbound=0) at /tmp/build/80754af9/python_1588882889832/work/Objects/typeobject.c:1518
#30 slot_nb_bool () at /tmp/build/80754af9/python_1588882889832/work/Objects/typeobject.c:6251
#31 0x000055c5e3d82d73 in PyObject_IsTrue () at /tmp/build/80754af9/python_1588882889832/work/Objects/object.c:1425
#32 0x000055c5e3db9418 in builtin_any () at /tmp/build/80754af9/python_1588882889832/work/Python/bltinmodule.c:427
#33 0x000055c5e3deaabd in _PyMethodDef_RawFastCallKeywords () at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:648
#34 0x000055c5e3deadb1 in _PyCFunction_FastCallKeywords (func=0x7f2a8a6ffaa0, args=<optimized out>, nargs=<optimized out>, kwnames=<optimized out>) at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:734
#35 0x000055c5e3e55e94 in call_function (kwnames=0x0, oparg=1, pp_stack=<synthetic pointer>) at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:4568
#36 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3124
#37 0x000055c5e3dea20b in function_code_fastcall (globals=<optimized out>, nargs=2, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:283
#38 _PyFunction_FastCallKeywords () at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:408
#39 0x000055c5e3e51be6 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:4616
#40 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3124
#41 0x000055c5e3dea20b in function_code_fastcall (globals=<optimized out>, nargs=3, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:283
#42 _PyFunction_FastCallKeywords () at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:408
#43 0x000055c5e3e56229 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:4616
#44 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3093
#45 0x000055c5e3d9ab00 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3930
#46 0x000055c5e3dea497 in _PyFunction_FastCallKeywords () at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:433
#47 0x000055c5e3e51be6 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:4616
#48 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3124
#49 0x000055c5e3dea20b in function_code_fastcall (globals=<optimized out>, nargs=7, args=<optimized out>, co=<optimized out>) at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:283
#50 _PyFunction_FastCallKeywords () at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:408
#51 0x000055c5e3e51be6 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:4616
#52 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3124
#53 0x000055c5e3d9a2b9 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3930
#54 0x000055c5e3d9b1d4 in PyEval_EvalCodeEx () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3959
#55 0x000055c5e3d9b1fc in PyEval_EvalCode (co=<optimized out>, globals=<optimized out>, locals=<optimized out>) at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:524
#56 0x000055c5e3e612ed in builtin_exec_impl.isra.12 (locals=0x7f2a8a677b40, globals=0x7f2a8a677b40, source=0x7f2a894b04b0) at /tmp/build/80754af9/python_1588882889832/work/Python/bltinmodule.c:1079
#57 builtin_exec () at /tmp/build/80754af9/python_1588882889832/work/Python/clinic/bltinmodule.c.h:283
#58 0x000055c5e3deab19 in _PyMethodDef_RawFastCallKeywords () at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:655
#59 0x000055c5e3deadb1 in _PyCFunction_FastCallKeywords (func=0x7f2a8a6ffe10, args=<optimized out>, nargs=<optimized out>, kwnames=<optimized out>) at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:734
#60 0x000055c5e3e55e94 in call_function (kwnames=0x0, oparg=2, pp_stack=<synthetic pointer>) at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:4568
#61 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3124
#62 0x000055c5e3d9a2b9 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3930
---Type <return> to continue, or q <return> to quit---
#63 0x000055c5e3dea435 in _PyFunction_FastCallKeywords () at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:433
#64 0x000055c5e3e51be6 in call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>) at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:4616
#65 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3124
#66 0x000055c5e3d9a2b9 in _PyEval_EvalCodeWithName () at /tmp/build/80754af9/python_1588882889832/work/Python/ceval.c:3930
#67 0x000055c5e3d9b3e5 in _PyFunction_FastCallDict () at /tmp/build/80754af9/python_1588882889832/work/Objects/call.c:376
#68 0x000055c5e3ea9ce7 in pymain_run_module () at /tmp/build/80754af9/python_1588882889832/work/Modules/main.c:355
#69 0x000055c5e3ebc60b in pymain_run_python (pymain=0x7ffe49a597e0) at /tmp/build/80754af9/python_1588882889832/work/Modules/main.c:2899
#70 pymain_main () at /tmp/build/80754af9/python_1588882889832/work/Modules/main.c:3442
#71 0x000055c5e3ebc6fc in _Py_UnixMain () at /tmp/build/80754af9/python_1588882889832/work/Modules/main.c:3477
#72 0x00007f2a89f30b97 in __libc_start_main (main=0x55c5e3d7b3a0 <main>, argc=7, argv=0x7ffe49a59938, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffe49a59928) at ../csu/libc-start.c:310
#73 0x000055c5e3e613c0 in _start () at ../sysdeps/x86_64/elf/start.S:103

Thread Info:
(gdb) info threads
  Id   Target Id         Frame 
* 1    Thread 0x7f2a8a739740 (LWP 154) "python3" 0x00007ffe49b79b44 in clock_gettime ()
  17   Thread 0x7f2a65791700 (LWP 1039) "jemalloc_bg_thd" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7f2a0c40a5f4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  18   Thread 0x7f2a65f92700 (LWP 1040) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  19   Thread 0x7f2a6a793700 (LWP 1041) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  20   Thread 0x7f2a6af94700 (LWP 1042) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  21   Thread 0x7f2a0b6a5700 (LWP 1043) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  22   Thread 0x7f2a0aea4700 (LWP 1044) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  23   Thread 0x7f2a0a6a3700 (LWP 1045) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  24   Thread 0x7f2a09ea2700 (LWP 1046) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  25   Thread 0x7f2a096a1700 (LWP 1047) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  26   Thread 0x7f2a08ea0700 (LWP 1048) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  27   Thread 0x7f2a03fff700 (LWP 1049) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  28   Thread 0x7f2a037fe700 (LWP 1050) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  29   Thread 0x7f2a02ffd700 (LWP 1051) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  30   Thread 0x7f2a027fc700 (LWP 1052) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  31   Thread 0x7f2a01ffb700 (LWP 1053) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  32   Thread 0x7f2a017fa700 (LWP 1054) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  33   Thread 0x7f2a00ff9700 (LWP 1055) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c5e8211dc4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  34   Thread 0x7f29dffff700 (LWP 1056) "jemalloc_bg_thd" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7f2a0c40a6c4) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  35   Thread 0x7f29d73ff700 (LWP 1057) "jemalloc_bg_thd" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7f2a0c40a790) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  36   Thread 0x7f29defff700 (LWP 1058) "jemalloc_bg_thd" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x7f2a0c40a860) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  39   Thread 0x7f29d5bdd700 (LWP 1922) "python3" 0x00007f2a8a032237 in accept4 (fd=10, addr=..., addr_len=0x7f29d5bdcdf8, flags=524288) at ../sysdeps/unix/sysv/linux/accept4.c:32
  40   Thread 0x7f284283f700 (LWP 1923) "python3" 0x00007f2a8a023bf9 in __GI___poll (fds=0x7f27b8000bd0, nfds=10, timeout=100) at ../sysdeps/unix/sysv/linux/poll.c:29
  48   Thread 0x7f2804c3f700 (LWP 1952) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c7ef6201fc) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  49   Thread 0x7f27f69e1700 (LWP 1953) "python3" 0x00007f2a8a30d9f3 in futex_wait_cancelable (private=<optimized out>, expected=0, futex_word=0x55c7ef61ffbc) at ../sysdeps/unix/sysv/linux/futex-internal.h:88
  50   Thread 0x7f281d23f700 (LWP 1961) "python3" 0x00007f2a8130faf1 in ?? () from /opt/conda/lib/python3.7/site-packages/torch/lib/libgomp-7c85b1e2.so.1
  51   Thread 0x7f281daf9700 (LWP 1962) "python3" 0x00007f2a8130faf1 in ?? () from /opt/conda/lib/python3.7/site-packages/torch/lib/libgomp-7c85b1e2.so.1
  52   Thread 0x7f28325fe700 (LWP 1963) "python3" 0x00007f2a8130faf1 in ?? () from /opt/conda/lib/python3.7/site-packages/torch/lib/libgomp-7c85b1e2.so.1
  53   Thread 0x7f281ca3e700 (LWP 1964) "python3" 0x00007f2a8130faf1 in ?? () from /opt/conda/lib/python3.7/site-packages/torch/lib/libgomp-7c85b1e2.so.1
  1279 Thread 0x7f27f59df700 (LWP 20593) "python3" 0x00007f2a8a3106d6 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f2738002d50) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
  1280 Thread 0x7f27f61e0700 (LWP 20594) "python3" 0x00007f2a8a3106d6 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f278c003860) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
  1281 Thread 0x7f27f3c7f700 (LWP 20595) "python3" 0x00007f2a8a3106d6 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f275c0047f0) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
  1282 Thread 0x7f2832dff700 (LWP 20596) "python3" 0x00007f2a8a3106d6 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f272c004600) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
  1283 Thread 0x7f28408bf700 (LWP 20597) "python3" 0x00007f2a8a3106d6 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f27a40050c0) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
  1284 Thread 0x7f283097f700 (LWP 20598) "python3" 0x00007f2a8a3106d6 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f2734001440) at ../sysdeps/unix/sysv/linux/futex-internal.h:205
  1285 Thread 0x7f27f347e700 (LWP 20599) "python3" 0x00007f2a8a3106d6 in futex_abstimed_wait_cancelable (private=0, abstime=0x0, expected=0, futex_word=0x7f273c003d70) at ../sysdeps/unix/sysv/linux/futex-internal.h:205