Well, I knew how to change my code to succeed with “all_reduce”, but I’m still a little comfused.
My code before:(fail)
from torch.nn.parallel import DistributedDataParallel as DDP

def main():
	processes = []
	for rank in range(world_size):
		p = Process(target=run, args=(rank, world_size))
		p.start()
		processes.append(p)
	for p in processes:
		p.join()

def run(local_rank, world_size):
    os.environ['MASTER_ADDR'] = '127.0.0.1'
    os.environ['MASTER_PORT'] = '29501'
	torch.cuda.set_device(local_rank)
    dist.init_process_group(world_size=world_size, rank=local_rank, backend='nccl')
	net = Net().cuda()
	criterion = ...
	optimizer = ...
	net = DDP(net, device_ids=[local_rank])
	for ep in range(2):
		# train
		net.train()
		for i, (images, labels) in enumerate(train_loader):
			optimizer.zero_grad()
			images = image.cuda()
			labels = labels.cuda()
			outputs = net(images)
			loss = criterion(outputs, labels)
			loss.backward()
			optimizer.step()
		# verification
		if local_rank != 0:
            val_acc = torch.tensor([0.0]).cuda()
		# only do verification on rank0
		if local_rank == 0:
			net.eval()
			# simplify verification simulation
			input = torch.randn(8, 3, 112, 112).cuda()
			with torch.no_grad():
				output = net(input)
			val_acc = torch.tensor([0.0009]).cuda()
		dist.all_reduct(val_acc)
		print(local_rank, val_acc)

main()

The code below shows:
1 tensor([0.0], device='cuda:1')
0 tensor([0.0009], device='cuda:0')

And, in epoch 2, local_rank 1 will hang up.
My solution is that, change the line “output = net(input)” to “output = net.module(input)”, it succeed.
0 tensor([0.0009], device='cuda:0')
1 tensor([0.0009], device='cuda:1')

So I want to know why this happen?