Hi @ptrblck,
I experienced the same problem
I tested on 1.2.0 and 1.3.1, both installed through conda from the pytorch channel - no improvements.
Below is the smallest example I could build to reproduce.
Initialising the model on multiple GPUs and using the ToDevice creates the problem.
Commenting out one or the other, or constructing the DataLoader with num_workers=0 solves it.
import torch
import torchvision
from torch.utils.data import Dataset, DataLoader


class DummyDataset(Dataset):
    def __init__(self, transform=None):
        super().__init__()
        self.transform = transform

    def __getitem__(self, idx):
        sample = torch.randn(2, 2, 2)

        if self.transform is not None:
            sample = self.transform(sample)
        return sample

    def __len__(self):
        return 10


class ToDevice(object):
    def __init__(self, device):
        self.device = device

    def __call__(self, sample):
        return sample.to(self.device)


class DummyModel(torch.nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, x):
        return torch.randn(2, 2, 2)

if __name__ == "__main__":
    device = torch.device("cuda")

    transform = None
    transform = torchvision.transforms.Compose([ToDevice(device)])  # A. commenting this removes exception

    dataset = DummyDataset(transform=transform)

    model = DummyModel().to(device)
    model = torch.nn.DataParallel(model)  # B. or commenting this - same story

    loader = DataLoader(dataset, num_workers=4)  # C. or setting num_workers=0
    for sample in loader:
        print(sample)
        break

It fails with error:
RuntimeError: Cannot re-initialize CUDA in forked subprocess. To use CUDA with multiprocessing, you must use the 'spawn' start method