In case you want some (sorry it is lengthy  ) code example:
import numpy as np
import torch
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
testset = torchvision.datasets.CIFAR10(root='./data', train=False,
                                       download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=64,
                                         shuffle=False, num_workers=2)
net = torchvision.models.resnet18()
net.fc = nn.Linear(512, 10)  # for CIFAR10
net.to(device)
criterion = nn.CrossEntropyLoss()

Case 1:
def test_epoch_case1(iterator, model, criterion):

    train_loss = 0
    all_y = []

    model.eval()

    for batch in iterator:
        X, y = batch[0], batch[1]
        X, y = X.to(device), y.to(device)

        with torch.no_grad():
            y_hat = model(X)

        loss = criterion(y_hat, y)
        train_loss += loss.item()

        all_y.append(y)

    y = torch.cat(all_y)
    print(f'This function returns collected train_loss: {train_loss} averaged by number of samples in y: {len(y)}')

    return train_loss / len(y)

test_epoch_case1(testloader, net, criterion)

prints:
This function returns collected train_loss: 362.682909488678 averaged by number of samples in y: 10000
0.0362682909488678

Case 2:
def test_epoch_case2(iterator, model, criterion):

    all_loss = []
    model.eval()

    for batch in iterator:
        X, y = batch[0], batch[1]
        X, y = X.to(device), y.to(device)

        with torch.no_grad():
            y_hat = model(X)

        loss = criterion(y_hat, y)
        all_loss.append(loss.item())

    print(f'This function returns collected train_loss: {np.sum(all_loss)} averaged by number of batches in dataloader: {len(iterator)} = {np.sum(all_loss) / len(iterator)}')

    return np.mean(all_loss)

test_epoch_case2(testloader, net, criterion)

prints:
This function returns collected train_loss: 362.682909488678 averaged by number of batches in dataloader: 157 = 2.310082226042535
2.310082226042535

Case 1 (corrected):
def test_epoch_case1_correct(iterator, model, criterion):

    train_loss = 0
    all_y = []

    model.eval()

    for batch in iterator:
        X, y = batch[0], batch[1]
        X, y = X.to(device), y.to(device)

        with torch.no_grad():
            y_hat = model(X)

        loss = criterion(y_hat, y)
        train_loss += loss.item()

        all_y.append(y)

    y = torch.cat(all_y)
    print(f'This function returns collected train_loss: {train_loss} averaged by number of batches in dataloader: {len(iterator)}')

    return train_loss / len(iterator)

test_epoch_case1_correct(testloader, net, criterion)

prints:
This function returns collected train_loss: 362.682909488678 averaged by number of batches in dataloader: 157
2.310082226042535

Hope it helps