Thousand thanks for the explanation! I modified my code following your suggestion and I provide my provisional solution here for comments.

running_loss = 0
running_len = 0
for inputs, labels in self.data_loader:
    self.optimizer.zero_grad()
    with torch.set_grad_enabled(phase=='train'):
        outputs = self.model(inputs)
        loss = self.loss(outputs, labels)
        if phase == 'train':
            loss.backward()
            self.optimizer.step()
            iteration_count+=1
    running_loss += loss.item()
    running_len += inputs.shape[0]
    torch.cuda.empty_cache()
    ##########
    is_next = torch.Tensor([self.data_loader.peek()])
    # is_next==True if the iterator has not reached the end, i.e., next loop is expected
    dist.all_reduce_multigpu(is_next, op=dist.ReduceOp.BAND)
    if not is_next: break
    ##########