Hey @Lewis_Liu,
Did you use fork or spawn?

Or if I want to periodically share the weights among processes, how to do it properly?

One solution is to create a multiprocessing queue, and pass that queue to child processes. Then, in the loop, use that queue to pass shared tensors. The test below can serve as an example:

  
      github.com
  
  
    pytorch/pytorch/blob/fc8bca094cc23a2394214c5cdbc8392a3d279e8c/test/distributed/test_c10d_spawn.py#L166-L184 1

@classmethod
def _test_allgather_process(
        cls, rank, filename, shared_tensors, world_size, init_pg, c2p, p2c):
    pg = init_pg(rank, filename, world_size)
    xs = [shared_tensors[rank]]
    ys = [[torch.zeros_like(xs[0]) for i in range(world_size)]]
    pg.allgather(ys, xs).wait()
    for i in range(world_size):
        c2p.put((rank, torch.ones(2, 2) * i, ys[0][i].to("cpu")))

    p2c.get()

@unittest.skipIf(not TEST_MULTIGPU, "At least 2 CUDA GPUS needed")
def test_shared_allgather_gloo(self):
    self._test_multiprocess(
        ProcessGroupShareTensorTest._test_allgather_process,
        [torch.ones(2, 2).to(i) * i for i in range(self.world_size)],
        ProcessGroupShareTensorTest._init_pg_gloo,
        self.world_size)