| { | |
| "add_prefix_space": false, | |
| "added_tokens_decoder": { | |
| "151643": { | |
| "content": "<|endoftext|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151644": { | |
| "content": "<|im_start|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151645": { | |
| "content": "<|im_end|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151646": { | |
| "content": "<B_SYS>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151647": { | |
| "content": "<B_USYS>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151648": { | |
| "content": "<C_Q>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151649": { | |
| "content": "<C_A>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151650": { | |
| "content": "<B_FUNC>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151651": { | |
| "content": "<B_CODE>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151652": { | |
| "content": "<B_APE>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": true, | |
| "special": true | |
| }, | |
| "151653": { | |
| "content": "<function_calling>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": true, | |
| "special": true | |
| }, | |
| "151654": { | |
| "content": "<calc_start>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": true, | |
| "special": true | |
| }, | |
| "151655": { | |
| "content": "<calc_end>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": true, | |
| "special": true | |
| }, | |
| "151656": { | |
| "content": "<inner_think>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": true, | |
| "special": true | |
| }, | |
| "151657": { | |
| "content": "<audio_start_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151658": { | |
| "content": "<audio_end_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151659": { | |
| "content": "<audio_pad_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151660": { | |
| "content": "<img_start_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151661": { | |
| "content": "<img_end_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151662": { | |
| "content": "<img_pad_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151663": { | |
| "content": "<img_newline_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151664": { | |
| "content": "<box_start_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151665": { | |
| "content": "<box_end_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151666": { | |
| "content": "<box_delim_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151667": { | |
| "content": "<ref_start_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151668": { | |
| "content": "<ref_end_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151669": { | |
| "content": "<img_delim_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151670": { | |
| "content": "<polygon_start_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151671": { | |
| "content": "<polygon_end_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151672": { | |
| "content": "<baichuan_pad_token>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151673": { | |
| "content": "<reserved_113>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151674": { | |
| "content": "<audio_delim_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151675": { | |
| "content": "<audiotext_start_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151676": { | |
| "content": "<audiotext_end_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151677": { | |
| "content": "<audiotext_pad_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151678": { | |
| "content": "<audiogen_start_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "151679": { | |
| "content": "<audiogen_end_baichuan>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| } | |
| }, | |
| "additional_special_tokens": [ | |
| "<|im_start|>", | |
| "<|im_end|>", | |
| "<B_SYS>", | |
| "<B_USYS>", | |
| "<C_Q>", | |
| "<C_A>", | |
| "<B_FUNC>", | |
| "<B_CODE>", | |
| "<B_APE>", | |
| "<function_calling>", | |
| "<calc_start>", | |
| "<calc_end>", | |
| "<inner_think>", | |
| "<audio_start_baichuan>", | |
| "<audio_end_baichuan>", | |
| "<audio_pad_baichuan>", | |
| "<img_start_baichuan>", | |
| "<img_end_baichuan>", | |
| "<img_pad_baichuan>", | |
| "<img_newline_baichuan>", | |
| "<box_start_baichuan>", | |
| "<box_end_baichuan>", | |
| "<box_delim_baichuan>", | |
| "<ref_start_baichuan>", | |
| "<ref_end_baichuan>", | |
| "<img_delim_baichuan>", | |
| "<polygon_start_baichuan>", | |
| "<polygon_end_baichuan>", | |
| "<baichuan_pad_token>", | |
| "<reserved_113>", | |
| "<audio_delim_baichuan>", | |
| "<audiotext_start_baichuan>", | |
| "<audiotext_end_baichuan>", | |
| "<audiotext_pad_baichuan>", | |
| "<audiogen_start_baichuan>", | |
| "<audiogen_end_baichuan>" | |
| ], | |
| "bos_token": null, | |
| "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<B_SYS>' + message['content']}}{% elif message['role'] == 'user_system' %}{{'<B_USYS>' + message['content']}}{% elif message['role'] == 'user' %}{{'<H_Q>' + message['content']}}{% elif message['role'] == 'assistant' %}{{'<H_A>' + message['content']}}{% elif message['role'] == 'function' %}{{'<B_FUNC>' + message['content']}}{% elif message['role'] == 'code' %}{{'<B_CODE>' + message['content']}}{% else %}{{ raise_exception('Invalid message role: ' + message['role']) }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{'<H_A>'}}{% endif %}", | |
| "clean_up_tokenization_spaces": false, | |
| "eos_token": "<|endoftext|>", | |
| "errors": "replace", | |
| "model_max_length": 8192, | |
| "pad_token": "<|endoftext|>", | |
| "split_special_tokens": false, | |
| "tokenizer_class": "Qwen2Tokenizer", | |
| "unk_token": null | |
| } | |