I had the same procedure while multiprocessing.
hi @e.pignatelli, @mcskwayrd
i recently solve it.
I hope this solution I propose will help you.
The cause of this problem is multiprocessing’s platform.
More specificly, Cuda’s library requests “spawn,” while Dataloader requests “fork.”
I think some dataloader problem and cada problem are related to this problem.
import torch
import torchvision
torch.multiprocessing.set_start_method('spawn')# good solution !!!!
from torch.utils.data import Dataset, DataLoader
import time

class DummyDataset(Dataset):
    def __init__(self, transform=None):
        super().__init__()
        self.transform = transform

    def __getitem__(self, idx):
        time.sleep(0.1);# !!!!!! In order to test, should be have virtual process time !!!!!!
        sample = torch.randn(2, 2, 2)
        if self.transform is not None:
            sample = self.transform(sample)
        return sample

    def __len__(self):
        return 100


class ToDevice(object):
    def __init__(self, device):
        self.device = device

    def __call__(self, sample):
        return sample.to(self.device)


class DummyModel(torch.nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, x):
        return torch.randn(2, 2, 2)

if __name__ == "__main__":
    device = torch.device("cuda")

    transform = None
    #transform = torchvision.transforms.Compose([ToDevice(device)])  # A. commenting this removes exception
    # ^ this problem line -> multiprocessing spawn platform is make new child process with the exception of handler

    #dataset = DummyDataset(transform=transform)
    dataset = DummyDataset()

    model = DummyModel().to(device)
    model = torch.nn.DataParallel(model)  # B. or commenting this - same story

    loader = DataLoader(dataset, num_workers=8)  # C. or setting num_workers=0
    
    ####################################################################
    # if num_worker > 0 then Dataloader should be required 'fork' platform.
    #  if do not change platform of multiprocessing from 'spawn' to 'fork' then you will see the error or very slow time.
    # https://pytorch.org/docs/stable/data.html
    try:
        torch.multiprocessing.set_start_method('fork',force=True)
    except RuntimeError:
        pass
    ####################################################################
    
    
    for epoch in range(2):
        start_millis = int(round(time.time() * 1000));
        
        for step,sample in enumerate(loader):
            sample.to(device)
            #       ^ transform here

        end_millis = int(round(time.time() * 1000));
        print(epoch+1,"epoch time: "+str(end_millis-start_millis)+" ms");