This shouldn’t be a problem, since you are creating separate Datasets for training and validation.
I created a small snippet, which might help you
# Create fake data
X_train = torch.randn(100, 3, 16, 16)
y_train = torch.Tensor(100).random_(0, 10).long()
X_val = torch.randn(50, 3, 16, 16)
y_val = torch.Tensor(50).random_(0, 10).long()

class MyDataset(Dataset):
    def __init__(self, X, y):
        self.data = X
        self.target = y
    
    def __getitem__(self, index):
        x = self.data[index]
        y = self.target[index]
        return x, y

    def __len__(self):
        return len(self.data)

def train():
    for data, target in train_loader:
        # Your training procedure...
        print data.shape
        print target.shape
    
def val():
    for data, target in val_loader:
        # Your validation procedure...
        print data.shape
        print target.shape

train_dataset = MyDataset(X_train, y_train)
val_dataset = MyDataset(X_val, y_val)
train_loader = DataLoader(train_dataset)
val_loader = DataLoader(val_dataset)

train()
val()