text_plain_output_1.png
from transformers import AutoTokenizer, BertModel

from transformers import AutoTokenizer, GPT2Model

import torch

import torch

text = 'Hello World!'

from transformers import AutoTokenizer, BertModel
tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')

text = 'i love cryptography, mathematics, and cybersecurity.'
tokenized_text = tokenizer.tokenize(text)

from transformers import AutoTokenizer, GPT2Model
tokenizer = AutoTokenizer.from_pretrained('gpt2')
model = GPT2Model.from_pretrained('gpt2')

text = 'i love cryptography, mathematics, and cybersecurity.'
tokenized_text = tokenizer.tokenize(text)

text = 'I love cryptography, mathematics, and cybersecurity.'
tokens = tokenizer.tokenize(text)

token_ids = tokenizer.convert_tokens_to_ids(tokens)

token_ids = tokenizer.encode(text)
tokens_from_encode = tokenizer.convert_ids_to_tokens(token_ids)

example_word = 'cyber'
example_token_id = tokenizer.convert_tokens_to_ids([example_word])[0]
example_embedding = model.embeddings.word_embeddings(torch.tensor([example_token_id]))

word_one = 'king'
word_two = 'queen'
one_token_id = tokenizer.convert_tokens_to_ids([word_one])[0]
one_embedding = model.embeddings.word_embeddings(torch.tensor([one_token_id]))
two_token_id = tokenizer.convert_tokens_to_ids([word_two])[0]
two_embedding = model.embeddings.word_embeddings(torch.tensor([two_token_id]))
cos = torch.nn.CosineSimilarity(dim=1)
cosine_similarity = cos(one_embedding, two_embedding)
euclidean_distance = torch.cdist(one_embedding, two_embedding)

word_three = 'man'
word_four = 'woman'
three_token_id = tokenizer.convert_tokens_to_ids([word_three])[0]
three_embedding = model.embeddings.word_embeddings(torch.tensor([three_token_id]))
four_token_id = tokenizer.convert_tokens_to_ids([word_four])[0]
four_embedding = model.embeddings.word_embeddings(torch.tensor([four_token_id]))
cos = torch.nn.CosineSimilarity(dim=1)
cosine_similarity = cos(three_embedding, four_embedding)
euclidean_distance = torch.cdist(three_embedding, four_embedding)
print(f"Cosine Similarity between '{word_three}' and '{word_four}': {cosine_similarity[0]}")
print(f"Euclidean Distance between '{word_three}' and '{word_four}': {euclidean_distance[0][0]}")