Hi, @rvarm1 .
I don’t use any distributed op in test, below is whole code of test. Only one line because I only use rank 0 to do test.
@torch.no_grad()
def test(model, val_loader, criterion, epoch, logger, collectors, gpu):
    top1_acc, top5_acc, loss_record = collectors
    top1_acc.reset()
    top5_acc.reset()
    loss_record.reset()

    model.eval()
    for meta_data in val_loader:
        data = meta_data[0].cuda(gpu, non_blocking=True)
        labels = meta_data[1].cuda(gpu, non_blocking=True)

        outputs = model(data)
        losses = criterion(outputs, labels)

        top1_acc.update(outputs, labels)
        top5_acc.update(outputs, labels)
        loss_record.update(losses)

    test_msg = 'Test Epoch {}, {}:{:.5}, {}:{:.5}, {}:{:.5}'.format(epoch, top1_acc.name, top1_acc.get(), top5_acc.name,
                                                                    top5_acc.get(), loss_record.name, loss_record.get())
    logger.info(test_msg)
    top1_acc.write_tb(name='Val Top1 Accuracy', iteration=epoch)
    top5_acc.write_tb(name='Val Top5 Accuracy', iteration=epoch)
    loss_record.write_tb(name='Val Loss', iteration=epoch)


And if I use any distributed op I should not pass that until all ranks do this op right?