I use the following code, but I think it may be wrong!
lr_broadcast = torch.tensor([0.0]).cuda()
if distribute:
if local_rank == 0:
lr_broadcast = torch.tensor([optimizer.state_dict()[‘param_groups’][0][‘lr’]]).cuda()
dist.all_reduce(lr_broadcast)
optimizer.state_dict()[‘param_groups’][0][‘lr’] = lr_broadcast.item()
print(local_rank, optimizer.state_dict()[‘param_groups’][0][‘lr’])