text_plain_output_1.png
from transformers import AutoTokenizer, BertModel

from transformers import AutoTokenizer, GPT2Model

text = 'Hello World!'

from transformers import AutoTokenizer, BertModel
tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')

text = 'i love cryptography, mathematics, and cybersecurity.'
tokenized_text = tokenizer.tokenize(text)

from transformers import AutoTokenizer, GPT2Model
tokenizer = AutoTokenizer.from_pretrained('gpt2')
model = GPT2Model.from_pretrained('gpt2')

text = 'i love cryptography, mathematics, and cybersecurity.'
tokenized_text = tokenizer.tokenize(text)
print(tokenized_text)