Hi, ptrblck. I’ve found the reason!
Please try the following code:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data as data
import numpy as np

torch.backends.cudnn.benchmark = False
# torch.backends.cudnn.deterministic = False
torch.backends.cudnn.deterministic = True # crash

class Net(nn.Module):
    def __init__(self):
        super().__init__()

        self.aspp = _AtrousSpatialPyramidPoolingModule(32, 16, output_stride=16)
        # Add 3 conv layers
        self.conv_layers = nn.Sequential(
            nn.Conv3d(16*5,
                      1,
                      kernel_size=3,
                      stride=1,
                      padding=1),
            nn.ReLU(),
        )

    def forward(self, x):
        x = self.aspp(x)
        x = self.conv_layers(x)

        return x

class _AtrousSpatialPyramidPoolingModule(nn.Module):

    def __init__(self, in_dim, reduction_dim=256, output_stride=16, rates=[6, 12, 18]):
        super(_AtrousSpatialPyramidPoolingModule, self).__init__()

        if output_stride == 8:
            rates = [2 * r for r in rates]
        elif output_stride == 16:
            pass
        else:
            raise 'output stride of {} not supported'.format(output_stride)

        self.features = []
        # 1x1 no dilation
        self.features.append(
            nn.Sequential(nn.Conv3d(in_dim, reduction_dim, kernel_size=1, bias=False),
                          nn.BatchNorm3d(reduction_dim),
                          nn.ReLU(inplace=True)))
        # dilation
        for r in rates:
            self.features.append(nn.Sequential(
                nn.Conv3d(in_dim, reduction_dim, kernel_size=3,
                          dilation=r, padding=r, bias=False),
                nn.BatchNorm3d(reduction_dim),
                nn.ReLU(inplace=True)
            ))
        self.features = torch.nn.ModuleList(self.features)

        # img level features
        self.img_avg_pooling = nn.Sequential(
            nn.AdaptiveAvgPool3d(1),
            nn.Conv3d(in_dim, reduction_dim, kernel_size=1, bias=False),
            nn.ReLU(inplace=True))


    def forward(self, x):
        x_size = x.size()

        img_features = self.img_avg_pooling(x)
        img_features = F.interpolate(img_features, x_size[2:],
                                     mode='trilinear',align_corners=True)
        out = img_features

        for f in self.features:
            y = f(x)
            out = torch.cat((out, y), 1)
        return out


class Dataset3D(data.Dataset):
    def __init__(self, size=(32, 144, 144, 144)):
        self.data = np.random.randn(16, 32, 144, 144, 144).astype(np.float32)
    def __getitem__(self, index):
        return self.data[index]
    def __len__(self):
        return 16

# Training dataset
training_dataset = Dataset3D()

trainloader = data.DataLoader(
    training_dataset,
    batch_size=1,
    shuffle=True)

net = Net().cuda(0)
optimizer = optim.SGD(net.parameters(), lr=0.01, momentum=0.9)

num_epochs = 10
for i in range(num_epochs):
    net.train()
    for j, inputs in enumerate(trainloader):

        inputs = inputs.to('cuda:0')

        optimizer.zero_grad()

        preds = net(inputs)

        print("Epoch {}\n".format(i + 1))


If I set dilation > 1 and torch.backends.cudnn.deterministic = True, it crashes.
On my Tesla V100-SXM2-32GB, it shows the above error. On my RTX 2080 Ti it gives the OOM error:
RuntimeError: CUDA out of memory. Tried to allocate 9.61 GiB (GPU 0; 10.73 GiB total capacity; 1.42 GiB already allocated; 8.50 GiB free; 837.00 KiB cached)

But if I set deterministic flag to be False, OR dilation = 1, it works on both machines.
I only take the deterministic flag literally and don’t know what’s going on in CUDA algorithms.
Is this a bug or I misused the flag?