Thank you very much for having a look at the script 
So it will sound silly but the problem wasn’t with these scripts but with the images supplied to the model. The mistake is in audiodataset.py that I uploaded below.
I was normalizing the frequency bins of the mel-spectrograms with as they did in the VoxCeleb paper:

def normalize_row_matrix(self, mat):

        mean_rows = mat.mean(axis=1)
        std_rows = mat.std(axis=1)
        normalized_array = (mat - mean_rows[:, np.newaxis]) / std_rows[:, np.newaxis]
        return normalized_array

However the image that is returned look without any apparent pattern which might make the training difficult for the model:
Here is a mel-spectrogram for a full segment (composed of multiple 3 seconds segments that are given as input in the model))

Here is the row normalized mel-spectrogram


"""
Define a class AudioDataset which take the folder of the training / test data as input 
and make normalized mel-spectrograms out of it
Labels are also one hot encoded
"""


from torch.utils.data import Dataset
from pydub import AudioSegment
from sklearn.preprocessing import LabelEncoder
from fs import open_fs
from fs.osfs import OSFS

import numpy as np
import librosa
import torch
import os

class AudioDataset(Dataset):
    def __init__(self, data_root, n_fft, hop_length, n_mels):
        self.data_root_fs = open_fs(data_root)
        self.samples = []
        self.n_fft = n_fft
        self.hop_length = hop_length
        self.n_mels = n_mels
        self.class_encode = LabelEncoder()
        self._init_dataset()

    def __len__(self):
        return len(self.samples)

    def __getitem__(self, idx):
        
        audio_filepath, label = self.samples[idx]
        
        with self.data_root_fs.open(audio_filepath, 'rb') as audio_fd:
            audio_file = AudioSegment.from_file(audio_fd)
        
        #audio_file = AudioSegment.from_file(audio_filepath)
        array_audio = np.array(audio_file.get_array_of_samples(), dtype=float)
        
        mel = self.to_mel_spectrogram(array_audio)
        #mel_norm = self.normalize_row_matrix(mel)
        mel_norm_tensor = torch.tensor(mel)
        mel_norm_tensor = mel_norm_tensor.unsqueeze(0)
        
        label_encoded = self.one_hot_sample(label)
        label_class = torch.argmax(label_encoded)
        
        return (mel_norm_tensor, label_class)
    
    
    def _init_dataset(self):
        
        folder_names = set()
        for match in self.data_root_fs.glob("*/*.wav"):
            folder = match.path.split('/')[-2]
            folder_names.add(folder)
            self.samples.append((match.path, folder))
        self.class_encode.fit(list(folder_names))
                
    def to_mel_spectrogram(self, x):

        sgram = librosa.stft(x, n_fft=self.n_fft, hop_length=self.hop_length)
        sgram_mag, _ = librosa.magphase(sgram)
        mel_scale_sgram = librosa.feature.melspectrogram(S=sgram_mag, sr=16000, n_mels=self.n_mels)
        mel_sgram = librosa.amplitude_to_db(mel_scale_sgram)
        return mel_sgram
    
    def normalize_row_matrix(self, mat):

        mean_rows = mat.mean(axis=1)
        std_rows = mat.std(axis=1)
        normalized_array = (mat - mean_rows[:, np.newaxis]) / std_rows[:, np.newaxis]
        return normalized_array
    
    def to_one_hot(self, codec, values):
        value_idxs = codec.transform(values)
        return torch.eye(len(codec.classes_))[value_idxs]

    def one_hot_sample(self, label):
        t_label = self.to_one_hot(self.class_encode, [label])
        return t_label