I am experimenting with cross-validation.
For splitting the train and validation set I am using the Subset class of PyTorch.
class Subset(Dataset):
    """
    Subset of a dataset at specified indices.
    Arguments:
        dataset (Dataset): The whole Dataset
        indices (sequence): Indices in the whole set selected for subset
    """
    def __init__(self, dataset, indices):
        self.dataset = dataset
        self.indices = indices

    def __len__(self):
        if self.indices.shape == ():
            print('this happens: Subset')
            return 1
        else:
            return len(self.indices)

    def __getitem__(self, idx):
        return self.dataset[self.indices[idx]]

num_val_samples = len(dataset)//k
for i in range(k):
    print('Processing fold: ', i + 1)
    """%%%% Initiate new model %%%%""" #in every fold
    # torch.cuda.manual_seed(1)
    # model = ResNet3D().to(device)
    # model.apply(weights_init)
    # Optimizer = optim.Adam(model.parameters(), lr=lr)
    # Criterion = nn.MSELoss()
    # Criterion = Criterion.cuda()
    Net, Optimizer, Criterion = build_model()
    """%%%% Split train and validation sets %%%%"""
    valid_idx = np.arange(len(dataset))[i * num_val_samples:(i + 1) * num_val_samples]
    print('Valid fold indices:', valid_idx + 1)
    train_idx = np.concatenate([np.arange(len(dataset))[:i * num_val_samples], np.arange(len(dataset))[(i + 1) * num_val_samples:]], axis=0)
    print('Train fold indices:', train_idx+1)
    train_dataset = Subset(dataset, train_idx)
    valid_dataset = Subset(dataset, valid_idx)
    train_loader = data.DataLoader(train_dataset, batch_size=batchsize, shuffle=True)
    valid_loader = data.DataLoader(train_dataset, batch_size=batchsize, shuffle=True) #same train_set to check

Without using the Optimizer.step() I am getting the following loss in 1 fold cv:

I did not do anything to the validation section. So it was as before:
        print('validating...')
        Net.eval()
        epoch_wise_loss = 0
        # for vBatch_idx, sample in enumerate(valid_loader):
        with torch.no_grad():
            for vBatch_idx, sample in enumerate(valid_loader):
            # Net.eval()
            # with torch.no_grad():
                mr = sample[0].float().to(device)
                # print("Valid input mean:", mr.mean())
                shimVal = sample[1].float().to(device)
                # print("Valid target mean:", shimVal.mean())
                out = Net(mr)
                Loss = Criterion(shimVal, out)
                print("Batch val loss:", Loss.item())
                batch_accuracy = accuracy(shimVal, out)
                epoch_wise_loss += Loss.item()
                mean_loss = epoch_wise_loss / (vBatch_idx + 1)

These are all without setting batchnorm to eval mode. The next is setting batchnorm to eval mode as following:
Net.train()
Net.apply(set_bn_eval)


Train and validation loss are exactly the same and increase as you can see from the graph above.