@thomwolf @lhoestq If you run the function below with any given text file (one sentence per line) , it will run the mapped function every time that you run the code for the same data and identical arguments.
def prepare_data(dataset_f: str,
                tokenizer: PreTrainedTokenizer,
                max_seq_length: int = None,
                batch_size: int = 64,
                num_workers: int = 0) -> Dict[str, DataLoader]:
   """Given an input file, prepare the train, test, validation dataloaders.
   :param dataset_f: input file
   :param tokenizer: pretrained tokenizer that will prepare the data, i.e. convert tokens into IDs
   :param max_seq_length: maximal sequence length. Longer sequences will be truncated
   :param batch_size: batch size for the dataloaders
   :param num_workers: number of CPU workers to use during dataloading. On Windows this must be zero
   :return: a dictionary containing train, test, validation dataloaders
   """
   max_seq_length = tokenizer.model_max_length if not max_seq_length else max_seq_length

   def preprocess(sentences: List[str]) -> Dict[str, Union[list, Tensor]]:
       """Preprocess the raw input sentences from the text file.
       :param sentences: a list of sentences (strings)
       :return: a dictionary of "input_ids"
       """
       tokens = [s.strip().split() for s in sentences]
       tokens = [t[:max_seq_length - 1] + [tokenizer.eos_token] for t in tokens]

       # The sequences are not padded here. we leave that to the dataloader in a collate_fn
       # (not included in this snippet for illustrative purposes)
       # That means: a bit slower processing, but a smaller saved dataset size
       encoded_d = tokenizer(tokens,
                             add_special_tokens=False,
                             is_pretokenized=True,
                             return_token_type_ids=False,
                             return_attention_mask=False)

       return {"input_ids": encoded_d["input_ids"]}

   dataset = Dataset.from_dict({"text": Path(dataset_f).read_text(encoding="utf-8").splitlines()})

   # 90% (train), 20% (test + validation)
   train_testvalid = dataset.train_test_split(test_size=0.2)
   # 10% of total (test), 10% of total (validation)
   test_valid = train_testvalid["test"].train_test_split(test_size=0.5)

   dataset = DatasetDict({"train": train_testvalid["train"],
                          "test": test_valid["test"],
                          "valid": test_valid["train"]})

   dataset = dataset.map(preprocess, input_columns=["text"], batched=True)
   dataset.set_format("torch", columns=["input_ids"])

   return {partition: DataLoader(ds,
                                 batch_size=batch_size,
                                 shuffle=True,
                                 num_workers=num_workers,
                                 pin_memory=True) for partition, ds in dataset.items()}

As a tokenizer I used the Dutch one (below), but the same issue exists with "bert-base-cased" so I don’t think the tokenizer is the issue.
tokenizer = AutoTokenizer.from_pretrained("wietsedv/bert-base-dutch-cased")
tokenizer.add_special_tokens({"eos_token": "[EOS]"})
prepare_data("path/to/data.txt", tokenizer)