I’m having the same issue and can confirm part of the analysis by @Pawel_Wiejacha as the SegFault happens on a Quadro P520 (Pascal architecture) with Pytorch 1.8.0 and Cuda 11.1. PyTorch is running in a virtualenv with Python 3.8.10. The gdb trace is appended below.
Some of the libraries I use aren’t available for conda, I tried creating the corresponding recipes, but ran into some upstream errors with the required dependencies. Therefor changing from wheels (available for all required libraries) to conda packages (not available for six of them) would come with considerable effort.
@ptrblck Any news on the issue? Or some other workaround maybe? (and btw thanks for your great work in the forums answering a lot of more or less coherent questions)
#0  0x00007fff951fb48f in at::cuda::getCurrentCUDABlasHandle() () from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cuda_cu.so
#1  0x00007fff951e97dd in void at::cuda::blas::gemm<float>(char, char, long, long, long, float, float const*, long, float const*, long, float, float*, long) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cuda_cu.so
#2  0x00007fff964058f0 in at::native::(anonymous namespace)::addmm_out_cuda_impl(at::Tensor&, at::Tensor const&, at::Tensor const&, at::Tensor const&, c10::Scalar, c10::Scalar) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cuda_cu.so
#3  0x00007fff96407092 in at::native::mm_cuda(at::Tensor const&, at::Tensor const&) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cuda_cu.so
#4  0x00007fff95240848 in at::(anonymous namespace)::(anonymous namespace)::wrapper_mm(at::Tensor const&, at::Tensor const&) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cuda_cu.so
#5  0x00007fff9524088f in c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<at::Tensor (at::Tensor const&, at::Tensor const&), &at::(anonymous namespace)::(anonymous namespace)::wrapper_mm>, at::Tensor, c10::guts::typelist::typelist<at::Tensor const&, at::Tensor const&> >, at::Tensor (at::Tensor const&, at::Tensor const&)>::call(c10::OperatorKernel*, at::Tensor const&, at::Tensor const&) () from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cuda_cu.so
#6  0x00007fff836be1b6 in at::Tensor c10::Dispatcher::call<at::Tensor, at::Tensor const&, at::Tensor const&>(c10::TypedOperatorHandle<at::Tensor (at::Tensor const&, at::Tensor const&)> const&, at::Tensor const&, at::Tensor const&) const [clone .constprop.4808] () from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#7  0x00007fff836c37df in at::mm(at::Tensor const&, at::Tensor const&) () from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#8  0x00007fff84fa750c in torch::autograd::VariableType::(anonymous namespace)::mm(at::Tensor const&, at::Tensor const&) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#9  0x00007fff84fa7a7f in c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<at::Tensor (at::Tensor const&, at::Tensor const&), &torch::autograd::VariableType::(anonymous namespace)::mm>, at::Tensor, c10::guts::typelist::typelist<at::Tensor const&, at::Tensor const&> >, at::Tensor (at::Tensor const&, at::Tensor const&)>::call(c10::OperatorKernel*, at::Tensor const&, at::Tensor const&) () from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#10 0x00007fff83b103c6 in at::Tensor c10::Dispatcher::call<at::Tensor, at::Tensor const&, at::Tensor const&>(c10::TypedOperatorHandle<at::Tensor (at::Tensor const&, at::Tensor const&)> const&, at::Tensor const&, at::Tensor const&) const [clone .constprop.1487] () from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#11 0x00007fff83b145ef in at::Tensor::mm(at::Tensor const&) const () from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#12 0x00007fff859e7421 in torch::autograd::generated::details::mm_mat2_backward(at::Tensor const&, at::Tensor const&, c10::ArrayRef<long>, c10::ArrayRef<long>, c10::Scalar const&) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#13 0x00007fff84dd37ae in torch::autograd::generated::MmBackward::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#14 0x00007fff85479361 in torch::autograd::Node::operator()(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#15 0x00007fff8547516b in torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&, std::shared_ptr<torch::autograd::ReadyQueue> const&) () from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#16 0x00007fff85475d8f in torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#17 0x00007fff8546d569 in torch::autograd::Engine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_cpu.so
#18 0x00007ffff5bef213 in torch::autograd::python::PythonEngine::thread_init(int, std::shared_ptr<torch::autograd::ReadyQueue> const&, bool) ()
   from /home/mbeeking/.local/share/virtualenvs/traveltime-prediction-UXvvem08/lib/python3.8/site-packages/torch/lib/libtorch_python.so
#19 0x00007ffff6c043c4 in std::execute_native_thread_routine (__p=0x5555c5f5a330) at /build/gcc/src/gcc/libstdc++-v3/src/c++11/thread.cc:82
#20 0x00007ffff7f4a259 in start_thread () from /usr/lib/libpthread.so.0
#21 0x00007ffff7d235e3 in clone () from /usr/lib/libc.so.6