Still couldn’t figure out how to take ownership of the pointer since they store the Tensor internally, but I did what I mentioned above and modified by ManagedMemory to have a custom deallocation func.
{
        at::Tensor a = at::randn({50, 50, 50});
        int tensorDataSize = std::accumulate(a.sizes().begin(), a.sizes().end(), 1, std::multiplies<int>()) * sizeof(float);
        DLManagedTensor * dlPackTensor = at::toDLPack(a);
        ManagedMemory managedMemory((char *) dlPackTensor->dl_tensor.data, tensorDataSize, CPU, true);
        managedMemory.deallocationFunc = [&]() { dlPackTensor->deleter(dlPackTensor); };
}