eqy:

gdb --args python my_script.py


Here is the backtrace output
Thread 19 "python" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7ffeb98e6700 (LWP 94700)]
0x00007ffebb50ff05 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
(gdb) backtrace
#0  0x00007ffebb50ff05 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#1  0x00007ffebb448c07 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#2  0x00007ffebb573f7d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#3  0x00007ffebb420e6d in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#4  0x00007ffebb4213ff in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#5  0x00007ffebb327f05 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#6  0x00007ffebb328052 in ?? () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#7  0x00007ffebb4ee87d in cuMemsetD8Async () from /usr/lib/x86_64-linux-gnu/libcuda.so.1
#8  0x00007ffefde3bd2e in ?? () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libcudart-6d56b25a.so.11.0
#9  0x00007ffefde1989b in ?? () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libcudart-6d56b25a.so.11.0
#10 0x00007ffefde57311 in cudaMemsetAsync ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libcudart-6d56b25a.so.11.0
#11 0x00007fff257dd155 in scaleFilter4d(cudnnContext*, cudnnFilter4dStruct*, void*, void const*) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#12 0x00007fff25abbc8b in cudnn::cnn::Wgrad2dAlgo0Engine<float, float, float>::execute_internal_impl(cudnn::backend::VariantPack const&, CUstream_st*) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#13 0x00007fff250386f3 in cudnn::cnn::EngineInterface::execute(cudnn::backend::VariantPack const&, CUstream_st*) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#14 0x00007fff25735750 in cudnn::cnn::EngineContainer<(cudnnBackendEngineName_t)2020, 4096ul>::execute_internal_impl(cudnn::backend::VariantPack const&, CUstream_st*) () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
---Type <return> to continue, or q <return> to quit---
#15 0x00007fff250386f3 in cudnn::cnn::EngineInterface::execute(cudnn::backend::VariantPack const&, CUstream_st*) ()

   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so


#16 0x00007fff254aeebc in cudnn::cnn::AutoTransformationExecutor::execute_pipeline(cudnn::cnn::ConvolutionEngine&, cudnn::backend::VariantPack const&, CUstream_st*) const () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so

#17 0x00007fff2575c4f1 in cudnn::cnn::GeneralizedConvolutionEngine<cudnn::cnn::EngineContainer<(cudnnBackendEngineName_t)2020, 4096ul> >::execute_internal_impl(cudnn::backend::VariantPack const&, CUstream_st*) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#18 0x00007fff250386f3 in cudnn::cnn::EngineInterface::execute(cudnn::backend::VariantPack const&, CUstream_st*) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#19 0x00007fff254330db in cudnn::backend::execute(cudnnContext*, cudnn::backend::ExecutionPlan&, cudnn::backend::VariantPack&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#20 0x00007fff2572719d in cudnn::backend::EnginesAlgoMap<cudnnConvolutionBwdFilterAlgo_t, 7>::execute_wrapper(cudnnContext*, cudnnConvolutionBwdFilterAlgo_t, cudnn::backend::ExecutionPlan&, cudnn::backend::VariantPack&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#21 0x00007fff25726d31 in cudnn::backend::cudnnConvolutionBackwardFilter(cudnnContext*, void const*, cudnnTensorStruct const*, void const*, cudnnTensorStruct const*, void const*, cudnnConvolutionStruct const*, cudnnConvolutionBwdFilterAlgo_t, void*, unsigned long, void const*, cudnnFilterStruct const*, void*)
    () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#22 0x00007fff2504091a in cudnnConvolutionBackwardFilter ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#23 0x00007fff2403bfbd in at::native::raw_cudnn_convolution_backward_weight_out_32bit(at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef---Type <return> to continue, or q <return> to quit---
<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool)::{lambda(cudnnConvolutionBwdFilterAlgoPerf_t const&)#1}::operator()(cudnnConvolutionBwdFilterAlgoPerf_t const&) const () from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#24 0x00007fff2403f01a in at::native::raw_cudnn_convolution_backward_weight_out_32bit(at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#25 0x00007fff2403fcaf in at::native::raw_cudnn_convolution_backward_weight_out(at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#26 0x00007fff240390fa in at::native::cudnn_convolution_backward_weight(char const*, c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#27 0x00007fff240397db in at::native::cudnn_convolution_backward_weight(c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#28 0x00007fff8637f6e2 in at::(anonymous namespace)::(anonymous namespace)::wrapper_cudnn_convolution_backward_weight(c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
#29 0x00007fff8637f771 in c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<at::Tensor (c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool), &at::(anonymous namespace)::(anonymous namespace)::wrapper_cudnn_convolution_backward_weight>, at::Tensor, c10::guts::typelist::typelist<c10::ArrayRef<long>, at::Tenso---Type <return> to continue, or q <return> to quit---
r const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool> >, at::Tensor (c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool)>::call(c10::OperatorKernel*, c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
#30 0x00007fff749d0485 in at::Tensor c10::Dispatcher::call<at::Tensor, c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool>(c10::TypedOperatorHandle<at::Tensor (c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool)> const&, c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool) const ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#31 0x00007fff74865ea1 in at::cudnn_convolution_backward_weight(c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#32 0x00007fff240335c9 in at::native::cudnn_convolution_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul>) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cpp.so
#33 0x00007fff8637f4f7 in at::(anonymous namespace)::(anonymous namespace)::wrapper_cudnn_convolution_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul>) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
#34 0x00007fff8637f58a in c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<std::tuple<at::Tensor, at::Tensor> (at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bo---Type <return> to continue, or q <return> to quit---
ol, bool, std::array<bool, 2ul>), &at::(anonymous namespace)::(anonymous namespace)::wrapper_cudnn_convolution_backward>, std::tuple<at::Tensor, at::Tensor>, c10::guts::typelist::typelist<at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul> > >, std::tuple<at::Tensor, at::Tensor> (at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul>)>::call(c10::OperatorKernel*, at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul>) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cuda_cu.so
#35 0x00007fff74866537 in at::cudnn_convolution_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul>) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#36 0x00007fff760e379f in torch::autograd::VariableType::(anonymous namespace)::cudnn_convolution_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul>) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#37 0x00007fff760e3eba in c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<std::tuple<at::Tensor, at::Tensor> (at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul>), &torch::autograd::VariableType::(anonymous namespace)::cudnn_convolution_backward>, std::tuple<at::Tensor, at::Tensor>, c10::guts::typelist::typelist<at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul> > >, std::tuple<at::Tensor, at::Tensor> (at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul>)>::call(c10::OperatorKernel*, at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul>) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
---Type <return> to continue, or q <return> to quit---
#38 0x00007fff74866537 in at::cudnn_convolution_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::ArrayRef<long>, long, bool, bool, bool, std::array<bool, 2ul>) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#39 0x00007fff75f4420c in torch::autograd::generated::CudnnConvolutionBackward::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#40 0x00007fff765be771 in torch::autograd::Node::operator()(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#41 0x00007fff765ba57b in torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&, std::shared_ptr<torch::autograd::ReadyQueue> const&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#42 0x00007fff765bb19f in torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#43 0x00007fff765b2979 in torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_cpu.so
#44 0x00007fffe6d3c163 in torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) ()
   from /home/ubuntu/anaconda3/envs/pytorch_latest_p37/lib/python3.7/site-packages/torch/lib/libtorch_python.so
#45 0x00007fffe7d486df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#46 0x00007ffff7bbb6db in start_thread (arg=0x7ffeb98e6700) at pthread_create.c:463
#47 0x00007ffff6f3771f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Output of nvidia-smi is
ri Jun 11 19:14:32 2021       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.119.03   Driver Version: 450.119.03   CUDA Version: 11.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla M60           On   | 00000000:00:1E.0 Off |                    0 |
| N/A   37C    P0    38W / 150W |      0MiB /  7618MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+