text_plain_output_1.png
from transformers import AutoTokenizer

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained('gpt2')

example = 'def gcd(a, b):\n    """Computes gcd(a,b) via the euclidean algorithm."""\n    while b > 0:\n        a,b = b, a%b\n    return a'
tokens = tokenizer.tokenize(example)
print(tokens)
print(len(tokens))