I was wondering if you would want to manipulate the result after you tokenize the text, as below:
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(‘bert-base-uncased’)
encoded_sent = tokenizer.encode_plus (text=‘cat dog human’, truncation=False,  padding=False)
print(‘Before Truncation: \n’)
print(encoded_sent)
print(tokenizer.convert_ids_to_tokens(encoded_sent[‘input_ids’]))
print(’\nAfter truncation: \n’)
#If maxlen greater than text size, use ‘PAD’ token
#If maxlen lesser than text size, truncate the first token
#Remember, we reserve two tokens for [CLS] and [SEP] as we use BERT here
maxlen =10
ids = encoded_sent[‘input_ids’]
if len(ids)>=maxlen:
ids = [ids[0]] + ids[2:maxlen] + [102]
else:
ids = ids + ([0] * (maxlen-len(ids)))
encoded_sent[‘input_ids’]=ids
ids = encoded_sent[‘token_type_ids’]
if len(ids)>=maxlen:
ids = [ids[0]] + ids[2:maxlen] + [0]
else:
ids = ids + ([0] * (maxlen-len(ids)))
encoded_sent[‘token_type_ids’]=ids
ids = encoded_sent[‘attention_mask’]
if len(ids)>=maxlen:
ids = [ids[0]] + ids[2:maxlen] + [1]
else:
ids = ids + ([0] * (maxlen-len(ids)))
encoded_sent[‘attention_mask’]=ids
print(encoded_sent)
print(tokenizer.convert_ids_to_tokens(encoded_sent[‘input_ids’]))