Spaces:

xu-song
/

tokenizer-arena

Running

xu-song commited on Jul 28, 2024

Commit

83a2e6b

1 Parent(s): 4cbb545

update

Files changed (1) hide show

vocab.py CHANGED Viewed

@@ -386,7 +386,8 @@ assert len(set([config.name_or_path.split("/")[-1] for config in _all_tokenizer_
 class TokenizerFactory:
     def __init__(self):
-        self.all_tokenizer_configs = sorted(_all_tokenizer_config, key=lambda k: k.name_or_path)
         self.all_tokenizer_names = [config.name_or_path for config in self.all_tokenizer_configs]
         self.name_to_config_list = [
             {config.name_or_path: config for config in self.all_tokenizer_configs},

 class TokenizerFactory:
     def __init__(self):
+        # self.all_tokenizer_configs = sorted(_all_tokenizer_config, key=lambda k: k.name_or_path)
+        self.all_tokenizer_configs = sorted(_all_tokenizer_config, key=lambda k: k.name_display)
         self.all_tokenizer_names = [config.name_or_path for config in self.all_tokenizer_configs]
         self.name_to_config_list = [
             {config.name_or_path: config for config in self.all_tokenizer_configs},