Can you try this way
if args.local_rank == 0:
    top1, top5 = test(net, testloader, criterion, False)
    top1 = torch.tensor(top1).cuda(args.local_rank)
else:
    top1 = torch.tensor(0.).cuda(args.local_rank)
torch.distributed.broadcast(top1,src=0, async_op=False)
print("local rank:{}, top1:{}".format(args.local_rank, top1.item()))