Below is an example of putting a fastText model into a dataset / dataloader.

  
      github.com
  
  
    dhpollack/fellowshipai/blob/master/data/europarl.py#L137 10

    samples = torch.take(sentence, sample_idxes)  # num_samples x sample_len
    # make labels for samples
    labels = [self.label_encoder[label]] * num_samples
    labels = torch.LongTensor(labels)
    return samples, labels, num_samples
def shuffle_data(self):
    combined = list(zip(self.data, self.labels))
    random.shuffle(combined)
    self.data[:], self.labels[:] = zip(*combined)


class EUROPARLFT(data.Dataset):
splits = {"train": "txt_noxml/europarl.tokenized.all",
          "test": "test/europarl.tokenized.test",
          "split_train": "txt_noxml/europarl.tokenized.split.train",
          "split_valid": "txt_noxml/europarl.tokenized.split.valid",
          "dev_train": "txt_noxml/europarl.tokenized.trunc.train",
          "dev_test": "test/europarl.tokenized.trunc.test"}
splits_len = {"train": 10088220,
              "test": 21000,
              "split_train": 9988221,
              "split_valid": 100000,