Bert, NLP.
tokenizer.tokenize(marked_text) # Map the token strings to their vocabulary indeces. indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) # Display the words with their indeces. for tup in zip(tokenized_text, indexed_tokens): print('{:<12} {:>6,}'.format(tup[0], tup[1])) [CLS] 101 after