Hi,
I try to use RPC, but it seem to difficult.
My idea is that rank0 will call my_add function, and rank1 will do it and return the value for rank0.
"""RPC with Torch"""
"""run.py:"""
#!/usr/bin/env python
import os
import torch
import torch.distributed as dist
from torch.multiprocessing import Process
import torch.distributed.rpc as rpc

def my_add(a,b):
    return a+b

def run(rank, size):
    tensor = torch.zeros(1)
    if rank == 0:
        rpc.init_rpc("worker0", rank=0, world_size=size)
        ret = rpc.rpc_sync("worker1", my_add, args=(2,3))
        print(ret)
    else:
        rpc.init_rpc("worker1", rank=1, world_size=size)
    rpc.shutdown()
    #print('Rank ', rank, ' has data ', tensor[0])

def init_process(rank, size, fn, backend='gloo'):
    """ Initialize the distributed environment. """
    os.environ['MASTER_ADDR'] = '127.0.0.1'
    os.environ['MASTER_PORT'] = '29500'
    dist.init_process_group(backend, rank=rank, world_size=size)
    fn(rank, size)


if __name__ == "__main__":
    size = 2
    processes = []
    for rank in range(size):
        p = Process(target=init_process, args=(rank, size, run))
        p.start()
        processes.append(p)

    for p in processes:
        p.join()

I try to run but the error is as below
Process Process-45:
Process Process-46:
Traceback (most recent call last):
Traceback (most recent call last):
  File "/Users/cnphuong/opt/anaconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
    self.run()
  File "/Users/cnphuong/opt/anaconda3/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
    self.run()
  File "/Users/cnphuong/opt/anaconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "/Users/cnphuong/opt/anaconda3/lib/python3.7/multiprocessing/process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "<ipython-input-22-0e6d083442bd>", line 29, in init_process
    fn(rank, size)
  File "<ipython-input-22-0e6d083442bd>", line 29, in init_process
    fn(rank, size)
  File "<ipython-input-22-0e6d083442bd>", line 16, in run
    rpc.init_rpc("worker0", rank=0, world_size=size)
  File "<ipython-input-22-0e6d083442bd>", line 20, in run
    rpc.init_rpc("worker1", rank=1, world_size=size)
  File "/Users/cnphuong/opt/anaconda3/lib/python3.7/site-packages/torch/distributed/rpc/__init__.py", line 77, in init_rpc
    store, _, _ = next(rendezvous_iterator)
  File "/Users/cnphuong/opt/anaconda3/lib/python3.7/site-packages/torch/distributed/rpc/__init__.py", line 88, in init_rpc
    api._init_rpc_backend(backend, store, name, rank, world_size, rpc_backend_options)
  File "/Users/cnphuong/opt/anaconda3/lib/python3.7/site-packages/torch/distributed/rendezvous.py", line 172, in _env_rendezvous_handler
    store = TCPStore(master_addr, master_port, world_size, start_daemon, timeout)
  File "/Users/cnphuong/opt/anaconda3/lib/python3.7/site-packages/torch/distributed/rpc/api.py", line 283, in _init_rpc_backend
    rpc_backend_options=rpc_backend_options,
  File "/Users/cnphuong/opt/anaconda3/lib/python3.7/site-packages/torch/distributed/rpc/backend_registry.py", line 75, in init_backend
    return backend.value.init_backend_handler(*args, **kwargs)
RuntimeError: Address already in use
  File "/Users/cnphuong/opt/anaconda3/lib/python3.7/site-packages/torch/distributed/rpc/backend_registry.py", line 101, in _process_group_init_backend_handler
    "Default process group must not be initialized before init_rpc."
RuntimeError: Default process group must not be initialized before init_rpc.

Please help!
Thanks.