I have tested the dataloader’s test set by getting test set multiple times in the same run as well as across different runs, and there is no difference now. Here is the updated test now
def check_data_sanity():
    train, val, _, test1 = get_dataloaders(base_folder='/home/annus/Desktop/'
                                                      'projects/forest_cover_change/'
                                                      'eurosat/images/tif/',
                                          batch_size=16)

    train, val, _, test2 = get_dataloaders(base_folder='/home/annus/Desktop/'
                                                      'projects/forest_cover_change/'
                                                      'eurosat/images/tif/',
                                          batch_size=16)


    train, val, _, test3 = get_dataloaders(base_folder='/home/annus/Desktop/'
                                                      'projects/forest_cover_change/'
                                                      'eurosat/images/tif/',
                                          batch_size=16)

    def get_dict_diff(d1, d2):
        return len(set(d1.values()) - set(d2.values()))

    # compare on the same run
    print(get_dict_diff(test1, test2))
    print(get_dict_diff(test2, test3))
    print(get_dict_diff(test3, test1))

    # compare across runs
    import pickle
    with open('test1.pkl', 'rb') as ts1:
        test1_old = pickle.load(ts1)
    print(get_dict_diff(test2, test1_old))