Transformers, Pytorch, Machine Learning, Deep Dives, Data Science.
str_to_tokens(s): return [ord(z)-97+3 for z in s] @staticmethod def tokens_to_str(tokens): return "".join([chr(x+94) for x in tokens]) def __call__(self, sentence, max_length=None, pad=False): x = torch.tensor(self.str_to_tokens(sentence)) x = torch.cat([torch.tensor([SOS_IDX]), x,