Solved by gcramer23 in post #4 
              
              
                I am not seeing a problem when I do a basic example 
train_dataset = []
for _ in range(1000):
    train_dataset.append(torch.rand(4, 4))

def basic_sampler(rank, world_size):
    sampler = torch.utils.data.distributed.DistributedSampler(
    	train_dataset,
    	num_replicas=world_size,
    	rank=ra…