Since I had two classes in separate variables I ended up making the custom class.
class MyDataset(Dataset):
    def __init__(self, speech, silence):
        self.data = list(map(lambda x: (x, 1), speech)) + list(map(lambda x: (x, 0), silence))
        
    def __getitem__(self, index):
        return self.data[index]
    
    def __len__(self):
        return len(self.data)

train_ds = MyDataset(train_speech, train_silence)
train_dl = DataLoader(train_ds, shuffle=True, batch_size=1024)

Thanks for helping me get through this.