Thanks for your reply!
Here is my short code:
def main(rank, dev_id, args):
    torch.distributed.init_process_group(backend="nccl",
                                         init_method='tcp://localhost:22',
                                         world_size=args['num_devices'],
                                         rank=dev_id)
    model = mymodel.to(dev_id)
    optimizer = optim.Adam(model.parameters(), lr=args['lr'])
    for epochs:
        pred = model(inputs)
        loss = criterion(pred, label)
        optimizer.zero_grad()
        loss.backward()

        for param_group in optimizer.param_groups:
            for p in param_group['params']:
                if p.requires_grad and p.grad is not None:
                    # print(p.grad.data.shape, p.grad.data.device) Ps. We can get grad information here
                    dist.all_reduce(p.grad.data, op=dist.ReduceOp.SUM)
                    p.grad.data /= n_processes
        optimizer.step()
    torch.distributed.barrier()

mp = torch.multiprocessing.get_context('spawn')
for id, device_id in enumerate(devices):
        procs.append(mp.Process(target=main, args=(id, device_id, args), daemon=True))
        procs[-1].start()
for p in procs:
    p.join()

The pytorch version I’m using is 1.4.0 and all my codes are running in AWS instance.
Ps. The port of init_method can only be 22 which is strange otherwise it got Runtime Error like this:
File “/home/ubuntu/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/distributed/rendezvous.py”, line 120, in _tcp_rendezvous_handler
store = TCPStore(result.hostname, result.port, world_size, start_daemon)
RuntimeError: connect() timed out.
Thanks for your time and reading!