These are the transforms that I apply for training and validation:
transform_train = transforms.Compose([
    transforms.RandomAffine(5, translate=(0.1, 0.1)),
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])

transform_test = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])

I realized I made a mistake for inference by not using the same transforms. Applying transform_test on images before inference now gives me around 50% for a run on 200 random images, which is the figure I got after the first epoch during training.
This is how my code looks right now:
def classification(model, imgs):
    if torch.cuda.is_available():
        print(f"Using CUDA device {torch.cuda.get_device_name(0)}")
        print(DIV)
        device = torch.device("cuda:0")
    else:
        print("No CUDA device found, using CPU")
        print(DIV)
        device = torch.device("cpu")

    tensor_imgs = [transform_test(img).unsqueeze_(0) for img in imgs]
    pred = []
    with torch.no_grad():
        for img in tensor_imgs:
            img = img.to(device)
            model = model.to(device)
            output = model(img)
            pred.append(output.data.cpu().numpy().argmax())
    return pred

I didn’t quite understand your suggestion, should I save the images that I use for training and run inference on them for comparison? For the training part I am using DataLoader for both training and test datasets.