text_plain_output_1.png
from datasets import load_dataset

from transformers import AutoTokenizer

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained('gpt2')

example = 'def gcd(a, b):\n    """Computes gcd(a,b) via the euclidean algorithm."""\n    while b > 0:\n        a,b = b, a%b\n    return a'
tokens = tokenizer.tokenize(example)

from datasets import load_dataset
dataset = load_dataset('espejelomar/code_search_net_python_10000_examples')

def get_training_corpus(size=10):
    return (dataset['train'][i:i + size]['whole_func_string'] for i in range(0, len(dataset['train']), size))
training_corpus = get_training_corpus()

updated_tokenizer = tokenizer.train_new_from_iterator(training_corpus, 52000)