input_list = sorted(glob.glob(os.path.join(dataset_dir, "input/*.npy")))
output_list = sorted(glob.glob(os.path.join(dataset_dir, "output/*.npy")))

X_train, X_valid, y_train, y_valid = train_test_split(input_list, output_list, test_size=0.3, random_state=42)
scaler = pickle.load(open(scaler_path, 'rb'))

X_train, X_valid, y_train, y_valid = train_test_split(input_list, output_list, test_size=0.3, random_state=42)
scaler = pickle.load(open(scaler_path, 'rb'))

train_dataset = RFFullDataset3d(X_train, y_train, scaler)
valid_dataset = RFFullDataset3d(X_valid, y_valid, scaler)

dataloader_train = DataLoader(train_dataset, 32, shuffle=True)
dataloader_valid = DataLoader(valid_dataset, 32, shuffle=True)

Here is how I split and load the train/valid data, I tried different random state and they all have the same issue.