Oh, yes. Sorry, I was running several codes today simultaneously. that’s why I didn’t notice this mistake.
Hey, I corrected the mistake, it wasn’t the problem though. Here you can see the traceback error. Actually I want to run the model on 4 gpus and I declared it as nproc=4, but I do not know if I should add something else to my code or not? At the moment it only reads one gpu.
class Model(nn.Module):
    # Our model
    def __init__(self):
        super(Model, self).__init__()
        self.fc1 = nn.Conv2d(1,10,3)
        self.bn1 = nn.BatchNorm2d(10)
        self.fc2= nn.Conv2d(10,20,3)
        self.bn2 = nn.BatchNorm2d(20)
        self.fc3= nn.Linear(11520,10) 
    def forward(self,x):
        print(f'inout_size: {x.size()}')
        x = F.relu(self.fc1(x))
        x = self.bn1(x)
        x = F.relu(self.fc2(x))
        x = self.bn2(x)
        x = x.view(x.size(0),-1)
        x = self.fc3(x)
        print(f'output_size: {x.size()}')
        return(x)
########################################    
def train(gpu):
    rank = gpu 
    dist.init_process_group(backed = 'nccl',
                           init_method = 'env://',
                           world_size= 4,
                           rank=rank)
    torch.manual_seed(0)
    model = Model()  
    torch.cuda.set_device(gpu)
    model= model.to(gpu)
    optimizer = optim.Adam(model.parameters(),lr=0.1)
    lr_sch = lr_scheduler.StepLR(optimizer,step_size=2,gamma=0.1)
    criterion = nn.CrossEntropyLoss().to(gpu)
    ######################################
    model = nn.DistributedDataParallel(model, device_ids = [gpu])
    #####################################
    mnist =torchvision.datasets.MNIST('./data',train= True,download=True,
                                      transform =transforms.ToTensor())
    ####################################
    train_sampler = torch.utils.data.distributed.DistributedSampler(mnist,
                                                                    num_replicas=4,
                                                                    rank = rank)
    ###################################
    dataloader = DataLoader(mnist,batch_size=32,num_workers =4,pin_memory=True,
                                               sampler = train_sampler)
    #####################################
    for epoch in range(10):
        total_loss =0
        for X,y in dataloader:   
            X= X.to(gpu)
            y = y.long().to(gpu)
            pred = model(X)
            loss = criterion(pred,y)
            t_loss+= loss.item()
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
        print(f'Loss: {t_loss/len(dataloader)}')
def main():
    os.environ['MASTER_ADDR'] = '172.20.24.55' ### the IP of vm_gpu02
    os.environ['MASTER_PORT'] = '9000'
    mp.spawn(train,nprocs=4)
if __name__=='__main__': 
    main()

Exception                                 Traceback (most recent call last)
<ipython-input-10-e18ebd33df91> in <module>
      1 if __name__=='__main__':
      2 
----> 3     main()

<ipython-input-9-331de420a7b8> in main()
      5     os.environ['MASTER_PORT'] = '9000'
      6 
----> 7     mp.spawn(train,nprocs=4)

~/anaconda3/lib/python3.7/site-packages/torch/multiprocessing/spawn.py in spawn(fn, args, nprocs, join, daemon, start_method)
    198                ' torch.multiprocessing.start_process(...)' % start_method)
    199         warnings.warn(msg)
--> 200     return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')

~/anaconda3/lib/python3.7/site-packages/torch/multiprocessing/spawn.py in start_processes(fn, args, nprocs, join, daemon, start_method)
    156 
    157     # Loop on join until it returns True or raises an exception.
--> 158     while not context.join():
    159         pass
    160 

~/anaconda3/lib/python3.7/site-packages/torch/multiprocessing/spawn.py in join(self, timeout)
    111                 raise Exception(
    112                     "process %d terminated with exit code %d" %
--> 113                     (error_index, exitcode)
    114                 )
    115 

Exception: process 2 terminated with exit code 1