StevanCakic:

import torch, os

def test_nccl_ops():
    num_gpu = 2
    print("NCCL init before spawn")
    import torch.multiprocessing as mp
    dist_url = "file:///tmp/nccl_tmp_file"
    mp.spawn(_test_nccl_worker, nprocs=num_gpu, args=(num_gpu, dist_url), daemon=False)
    print("NCCL init succeeded.")


def _test_nccl_worker(rank, num_gpu, dist_url):
    import torch.distributed as dist

    dist.init_process_group(backend="NCCL", init_method=dist_url, rank=rank, world_size=num_gpu)
    dist.barrier()
    print("Worker after barrier")

if __name__ == "__main__":
    test_nccl_ops()



Hey @StevanCakic, for the above script, have you tried setting CUDA_VISIBLE_DEVICES for each spawned process before any torch operation, so that each process only sees one GPU? You can also try torch.cuda.set_device(), but I would recommend CUDA_VISIBLE_DEVICES, as with that you can know for sure that each process is exclusively using the expected device.