I’m facing this problem in my code, when returning the torch::Tensor type.
To reproduce the problem, I have setup the simple extension available as part of the pytorch test code available here - https://github.com/pytorch/pytorch/blob/master/test/cpp_extensions/extension.cpp 1
My environment
Python 3.6.9
OS: Ubuntu 18.04.3 LTS
g++: 7.5.0
torch: 1.6.0
Simple cpp extension, has the following code,
#include <torch/extension.h>

// test include_dirs in setuptools.setup with relative path
// #include <tmp.h>

torch::Tensor sigmoid_add(torch::Tensor x, torch::Tensor y)
{
    return x.sigmoid() + y.sigmoid();
}

struct MatrixMultiplier
{
    MatrixMultiplier(int A, int B)
    {
        tensor_ =
            torch::ones({A, B}, torch::dtype(torch::kFloat64).requires_grad(true));
        test = "Hello";
    }
    torch::Tensor forward(torch::Tensor weights)
    {
        return tensor_.mm(weights);
    }
    torch::Tensor get() const
    {
        return tensor_;
    }

    std::string getstr() const
    {
        return test;
    }

private:
    torch::Tensor tensor_;
    std::string test;
};

bool function_taking_optional(c10::optional<torch::Tensor> tensor)
{
    return tensor.has_value();
}

PYBIND11_MODULE(TORCH_EXTENSION_NAME, m)
{
    m.def("sigmoid_add", &sigmoid_add, "sigmoid(x) + sigmoid(y)");
    m.def(
        "function_taking_optional",
        &function_taking_optional,
        "function_taking_optional");
    py::class_<MatrixMultiplier>(m, "MatrixMultiplier")
        .def(py::init<int, int>())
        .def("forward", &MatrixMultiplier::forward)
        .def("get", &MatrixMultiplier::get)
        .def("getstr", &MatrixMultiplier::getstr);
}

setup.py file
from setuptools import setup, Extension
from torch.utils import cpp_extension

setup(name="test_ext",
    ext_modules=[cpp_extension.CppExtension(
        name='detr',
        sources=['test_module.cpp']
        )
    ],
    cmdclass={'build_ext': cpp_extension.BuildExtension}
)

tests.py file:

import test_ext
import unittest

class TestLib(unittest.TestCase):
    def test_matrix_mal(self):
        mm = test_ext.MatrixMultiplier(3, 4)
        mm.get()


traceback error:-

[New Thread 0x7fff943f2700 (LWP 1625)]
[New Thread 0x7fff93bf1700 (LWP 1626)]
[New Thread 0x7fff8d3f0700 (LWP 1628)]

Thread 1 "python" received signal SIGSEGV, Segmentation fault.
0x00007fffe550b76a in THPVariable_NewWithVar(_typeobject*, at::Tensor) () from /mnt1/projects/venv/lib/python3.6/site-packages/torch/lib/libtorch_python.so

Accessing the binding after building and installation step:
pip uninstall -y test_ext && python setup.py install --verbose && python -m unittest tests.py

Any idea on this ?, I was trying on the nightly version also, seems it’s not related to torch, may be python version or other thing.