Realised I forgot to add the code for the custom dataset. Here it is:
class RoadSignDataset(Dataset):

    def __init__(self, csv_path):
        self.to_tensor = transforms.ToTensor()
        #read the CSV file
        self.data_info = pd.read_csv(csv_path, header=None)
        #first column contains the image paths
        self.image_arr = np.asarray(self.data_info.iloc[:, 0])
        #second column is the labels
        self.label_arr = np.asarray(self.data_info.iloc[:, 7])
        #calculate len
        self.data_len = len(self.data_info.index)

    def __getitem__(self, index):
        #get image name from the pandas df
        single_image_name = '' + self.image_arr[index]
        #open image
        img_as_img = Image.open(single_image_name)

        #transform image to tensor
        img_as_tensor = self.to_tensor(img_as_img)

        #get label of the image based on the cropped pandas column
        single_image_label = self.label_arr[index]
        return(img_as_tensor, single_image_label)

    def __len__(self):
        return self.data_len