Could you please post a short code to introduce the instructions of it?
I have a machine with two GPUs, which means I want to use single process multi gpus.
I tried to use SyncBatchNorm, but failed, sadly like this …
It raise a “ValueError: SyncBatchNorm is only supported for DDP with single GPU per process”…!
But in docs of DDP 32, it says single-process multi-gpus.
import torch
import torch.nn as nn
class net(nn.Module):
    def __init__(self):
        super(net, self).__init__()
        self.convBlock = nn.Sequential(
            nn.Conv2d(3, 128, 3, 1, 1),
            nn.SyncBatchNorm(128),
            nn.ReLU(),
            nn.Conv2d(128, 512, 3, 1, 1),
            nn.SyncBatchNorm(512),
            nn.ReLU(),
            nn.Conv2d(512, 1, 3, 1, 1),
            nn.SyncBatchNorm(1),
            nn.ReLU()
        )
    def forward(self, x):
        x = self.convBlock(x)
        return x
torch.distributed.init_process_group(backend='nccl', init_method='tcp://127.0.0.1:12345', world_size=1, rank=0)

model = net().cuda()
model = nn.parallel.DistributedDataParallel(model, device_ids=[0, 1], output_device=0)
model = model
 optimizer = torch.optim.Adam(model.parameters())
mseloss = torch.nn.L1Loss()
for i in range(1000):
    x = torch.rand(10, 3, 224, 224)
    y = torch.rand(10, 1, 224, 224)
    x = x.cuda()
    y = y.cuda()
    out = model(x)
    optimizer.zero_grad()
    loss = mseloss(out, y)
    print(i, loss)
    loss.backward()
    optimizer.step()