I am  trying to successfully attempt facial recognition on a custom dataset of 48 classes with over 5000 images using Resnet50.
In the code below when I do NOT use transforms to create the image datasets, then i achieve 99% accuracy on training and validation set. After training, when I attempt to use the predict_image function below in order to classify a handful of images, i always seem to get the same label.  Would you know what the reason for this is? Furthermore, if do consider using transform to create the datasets, i achieve accuracies in the order of 0.04. I do not know why this is? I would appreciate if any one could help me?
from torchvision import datasets, transforms, models
import torch
from torch.utils.data import DataLoader, Dataset
import torchvision

from torch import Tensor
from torch.autograd import Variable

import torch.nn as nn
from sklearn.preprocessing import LabelEncoder
import numpy as np
import os
import cv2
from PIL import Image

from matplotlib import pyplot as plt


from sklearn.model_selection import train_test_split

#from sklearn.neural_network import MLPClassifier



from untilities import create_directory


from dataset_loader import DatasetLoader
from face_extractor import FaceExtracter
from data_augmentation import DataAugmentation



# loading dataset
base_dir = os.path.dirname(os.getcwd())
resources_absolute_path = os.path.join(base_dir, 'resources')
image_dir = os.path.join(resources_absolute_path, 'samples')

model_outputs_dir = os.path.join(resources_absolute_path,'model_outputs')

if(not os.path.exists(model_outputs_dir)):
    create_directory(model_outputs_dir)

# Create directory 'dataset' if it does not exist
labelled_dir = os.path.join(resources_absolute_path,'dataset', 'labelled3')
        
#unlabelled directory for exracted faces from group images/videos
unlabelled_dir = os.path.join(resources_absolute_path, 'dataset','unlabelled')         

##Pre-processing, detecting and extracting faces
if(not os.path.exists(os.path.join(resources_absolute_path, 'dataset'))):
    extracter = FaceExtracter(image_dir,resources_absolute_path)
    extracter.extract()
    augmenter = DataAugmentation()
    augmenter.augment(labelled_dir)
    #augmenter.remove_augmented_files(labelled_dir)


#loading extracted faces and labels
dataset = DatasetLoader(labelled_dir)
images,labels = dataset.load()

images_array = np.asarray(images)
labels_array = np.asarray(labels)

#del images,labels
labelencoder = LabelEncoder()

X_train, X_val, y_train, y_val = train_test_split(images_array, labels_array, test_size=0.2, random_state=1)
del images_array,labels_array

##label encode ytrainn and ytest
y_train_encoded = labelencoder.fit_transform(y_train)
y_val_encoded = labelencoder.transform(y_val)




########################################pytorch ########################################


import torchvision.transforms.functional as TF
class CustomTensorDataset(Dataset):
    """TensorDataset with support of transforms.
    """
    def __init__(self, tensors, transform=None):
        #assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
        self.tensors = tensors
        self.transform = transform

    def __getitem__(self, index):
        x = self.tensors[0][index]

        if self.transform:
            x = TF.to_pil_image(x)
            x = self.transform(x)

        y = self.tensors[1][index]

        return x, y

    def __len__(self):
        return self.tensors[0].size(0)
    
data_transforms = {
    'train': transforms.Compose([
        transforms.RandomResizedCrop(224),
        transforms.RandomRotation(degrees=15),
        transforms.ColorJitter(),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val': transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}

#image_datasets = {x: CustomTensorDataset(tensors=(eval("X_{}".format(x)), eval("y_{}".format(x)), transform=data_transforms[x] for x in ['train', 'val']}


image_datasets = {'train':CustomTensorDataset(tensors=(Tensor(X_train), Tensor(y_train_encoded)), transform=data_transforms['train']),
                  'val':CustomTensorDataset(tensors=(Tensor(X_val), Tensor(y_val_encoded)), transform=data_transforms['val']) }

dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=5,shuffle=True,num_workers=0)
              for x in ['train', 'val']}

dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}


###Visualising Dataset


import numpy as np

def imshow(image, ax=None, title=None, normalize=True):
    if ax is None:
        fig, ax = plt.subplots()
    image = image.permute(0,2,1)
    image = image.numpy().transpose((1, 2, 0))
    
    if normalize:
        mean = np.array([0.485, 0.456, 0.406])
        std = np.array([0.229, 0.224, 0.225])
        image = std * image + mean
        image = np.clip(image, 0, 1)

    ax.imshow(image)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.spines['left'].set_visible(False)
    ax.spines['bottom'].set_visible(False)
    ax.tick_params(axis='both', length=0)
    ax.set_xticklabels('')
    ax.set_yticklabels('')

    return ax

train_iter = iter(dataloaders['val'])
images, labels = train_iter.next()

fig, axes = plt.subplots(figsize=(12, 12), ncols=5)
print('training images')
for i in range(5):
    axe1 = axes[i] 
    imshow(images[i], ax=axe1, normalize=False)

print(images[0].size())



"""mean = 0.
std = 0.
nb_samples = len(X_train)
for data,_ in dataloaders['train']:
    batch_samples = data.size(0)
    data = data.view(batch_samples, data.size(1), -1)
    mean += data.mean(2).sum(0)
    std += data.std(2).sum(0)
mean /= nb_samples
std /= nb_samples

dataiter = iter(dataloaders['train'])
i, l = dataiter.next()"""


device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
import time
from torch import optim
from torch.optim import lr_scheduler
import copy
def train_model(model, criterion, optimizer, scheduler = None, num_epochs=25):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                # it-disables-dropout
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for inputs, labels in dataloaders[phase]:
                inputs = inputs.to(device)
                labels = labels.to(device)

                #inputs = inputs.permute(0,3,1,2)
                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels.long())

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.long().data)
            if phase == 'train' and scheduler:
                scheduler.step()

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))

            # deep copy the model
            if phase == 'val' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model



model_ft = models.resnet50(pretrained=True)

num_ftrs = model_ft.fc.in_features


# Here the size of each output sample is set to 48.
# Alternatively, it can be generalized to nn.Linear(num_ftrs, len(class_names)).
#model_ft.fc = nn.Linear(num_ftrs, len(set(y_train)))

####RESNET Classifier ################
model_ft.fc = nn.Sequential(
                      nn.Linear(num_ftrs, 256), 
                      nn.ReLU(), 
                      nn.Dropout(0.4),
                      nn.Linear(256, len(set(y_train))),                   
                      nn.LogSoftmax(dim=1))
model_ft = model_ft.to(device)
#model_ft.load_state_dict(torch.load(os.path.join(model_outputs_dir,'resnet2.pt')))


#model_vg.load_state_dict(torch.load(os.path.join(model_outputs_dir,'vgg162.pt')))
criterion = nn.CrossEntropyLoss()

#Optimization algorithm
optimizer = torch.optim.Adam(model_ft.parameters(), lr = 0.0001)# lr: learning rate

# Observe that all parameters are being optimized
optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.0001, momentum=0.9)

# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)

model_ft = train_model(model_ft, criterion, optimizer_ft,exp_lr_scheduler,5)

torch.save(model_ft.state_dict(), os.path.join(model_outputs_dir,'resnet2.pt'))




image = Image.open(os.path.join(labelled_dir,'09','IMG_6884.JPG'))

def predict_image(image,model):
    #modelX=torch.load('trained_models/facial_.pth')
    model.eval()

    
    test_transforms = transforms.Compose([transforms.Resize([224,224]),
                          transforms.Grayscale(num_output_channels=3),
                           transforms.ToTensor(),
                           ])

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    image_tensor = test_transforms(image).float()
    image_tensor = image_tensor.unsqueeze_(0)
    input = Variable(image_tensor)
    input = input.to(device)
    output = model(input)
    index = output.data.cpu().numpy().argmax()
    return index

print(labelencoder.inverse_transform([predict_image(image, model_ft)]))