Distributed Training, Model Parallelism, Data Parallelism, Pytorch, Aurorain.
world_size): ... # initialize the process group dist.init_process_group("tst", rank=rank, world_size=world_size) def cleanup(): dist.destroy_process_group() ### Step 2: define DDP modeling def dummy_init(rank, world_size): setup(rank, world_size) model = DummyModel().to(rank) ddp_model = DDP(model,