Hi @flyaway,
I can’t reproduce your problem, What version of tokenizers are you using ?
 from tokenizers import Tokenizer, models, pre_tokenizers, trainers
import tokenizers


print(tokenizers.__version__)
# 0.9.4


def build_BPE_tokenizer(
    train_files,
    output_dir,
    # name: str,
    vocab_size: int,
    min_frequency: int,
):

    tokenizer = Tokenizer(models.BPE())
    tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()

    trainer = trainers.BpeTrainer(
        vocab_size=vocab_size,
        min_frequency=min_frequency,
        special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>"],
    )

    tokenizer.train(trainer, train_files)
    tokenizer.model.save(output_dir)
    return tokenizer

# Test.txt contains the examples you gave
tokenizer = build_BPE_tokenizer(["test.txt"], "out", 100, 1)

print(tokenizer.encode("喜欢 打篮球 的 男生 喜欢 什么样 的 女生 ").tokens)
# Output is ['喜欢', '打篮球', '的', '男生', '喜欢', '什么样', '的', '女生']

Are you sure you were using the correct tokenizer ?