The next question is about the “Fast tokenizers’ special powers” section, and in particular, the algorithm to group entities.
Say the model decides to split (incorrectly) “Hugging Face” into two entities “Hugging” and “Face”. [This happened when I was trying out a smaller model.]
# # Hugging Face is one entity
# predictions = [0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 6, 6, 6, 0, 8, 0, 0]
# # Hugging Face is two entities
predictions = [0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 6, 6, 5, 0, 8, 0, 0]

Then “Face” is not included in the named entity results.
entity_group	score	    word	start     end
         PER    0.998169    Sylvain     11     18
         ORG    0.975004    Hugging     33     40
         LOC    0.993211    Brooklyn    49     57

I have a reprex to reproduce this edge case and propose a solution.


reprex
import numpy as np
import pandas as pd
import torch
from transformers import AutoModelForTokenClassification, AutoTokenizer

model_checkpoint = "dbmdz/bert-large-cased-finetuned-conll03-english"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)

example = "My name is Sylvain and I work at Hugging Face in Brooklyn."
inputs = tokenizer(example, return_tensors="pt")
outputs = model(**inputs)
probabilities = torch.nn.functional.softmax(outputs.logits, dim=-1)[0].tolist()

inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)
tokens = inputs_with_offsets.tokens()
offsets = inputs_with_offsets["offset_mapping"]

# # Hugging Face is one entity
# predictions = [0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 6, 6, 6, 0, 8, 0, 0]
# # Hugging Face is two entities
predictions = [0, 0, 0, 0, 4, 4, 4, 4, 0, 0, 0, 0, 6, 6, 5, 0, 8, 0, 0]

results = []

idx = 0
while idx < len(predictions):
    pred = predictions[idx]
    label = model.config.id2label[pred]

    # Original solution: skips over the second of two consecutive named entities.
    if label != "O":
        # Remove the B- or I-
        label = label[2:]
        start, _ = offsets[idx]

        # Grab all the tokens labeled with I-label
        all_scores = []
        while (
            idx < len(predictions)
            and model.config.id2label[predictions[idx]] == f"I-{label}"
        ):
            all_scores.append(probabilities[idx][pred])
            _, end = offsets[idx]
            idx += 1

    # # Proposed solution: recognizes both of two consecutive named entities.
    # if label != "O":
    #     # Remove the B- or I- prefix
    #     label = label[2:]
    #     start, end = offsets[idx]
    #     all_scores = [probabilities[idx][pred]]
    #
    #     # Grab all subsequent tokens with the same I-label, if any
    #     while (
    #         idx + 1 < len(predictions)
    #         and model.config.id2label[predictions[idx + 1]] == f"I-{label}"
    #     ):
    #         idx += 1
    #         all_scores.append(probabilities[idx][pred])
    #         _, end = offsets[idx]

        # Take the mean of all token scores in the grouped entity
        score = np.mean(all_scores).item()
        word = example[start:end]
        results.append(
            {
                "entity_group": label,
                "score": score,
                "word": word,
                "start": start,
                "end": end,
            }
        )
    idx += 1

pd.DataFrame(results)