| { | |
| "add_prefix_space": false, | |
| "added_tokens_decoder": { | |
| "100256": { | |
| "content": "<|_unuse_missing_100256|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100257": { | |
| "content": "<|endoftext|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100258": { | |
| "content": "<|fim_prefix|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100259": { | |
| "content": "<|fim_middle|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100260": { | |
| "content": "<|fim_suffix|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100261": { | |
| "content": "<|_unuse_missing_100261|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100262": { | |
| "content": "<|_unuse_missing_100262|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100263": { | |
| "content": "<|_unuse_missing_100263|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100264": { | |
| "content": "<|_unuse_missing_100264|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100265": { | |
| "content": "<|_unuse_missing_100265|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100266": { | |
| "content": "<|_unuse_missing_100266|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100267": { | |
| "content": "<|_unuse_missing_100267|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100268": { | |
| "content": "<|_unuse_missing_100268|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100269": { | |
| "content": "<|_unuse_missing_100269|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100270": { | |
| "content": "<|_unuse_missing_100270|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100271": { | |
| "content": "<|_unuse_missing_100271|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100272": { | |
| "content": "<|im_start|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100273": { | |
| "content": "<|im_end|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100274": { | |
| "content": "<|stop|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100275": { | |
| "content": "<|endofturn|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "100276": { | |
| "content": "<|endofprompt|>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110491": { | |
| "content": "<repo_name>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110492": { | |
| "content": "<file_sep>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110493": { | |
| "content": "<issue_start>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110494": { | |
| "content": "<issue_comment>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110495": { | |
| "content": "<issue_closed>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110496": { | |
| "content": "<jupyter_start>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110497": { | |
| "content": "<jupyter_text>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110498": { | |
| "content": "<jupyter_code>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110499": { | |
| "content": "<jupyter_output>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110500": { | |
| "content": "<jupyter_script>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110501": { | |
| "content": "<empty_output>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110502": { | |
| "content": "<code_to_intermediate>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110503": { | |
| "content": "<intermediate_to_code>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110504": { | |
| "content": "<pr>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110505": { | |
| "content": "<pr_status>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110506": { | |
| "content": "<pr_is_merged>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110507": { | |
| "content": "<pr_base>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110508": { | |
| "content": "<pr_file>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110509": { | |
| "content": "<pr_base_code>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110510": { | |
| "content": "<pr_diff>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110511": { | |
| "content": "<pr_diff_hunk>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110512": { | |
| "content": "<pr_comment>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110513": { | |
| "content": "<pr_event_id>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110514": { | |
| "content": "<pr_review>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110515": { | |
| "content": "<pr_review_state>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110516": { | |
| "content": "<pr_review_comment>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110517": { | |
| "content": "<pr_in_reply_to_review_id>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110518": { | |
| "content": "<pr_in_reply_to_comment_id>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110519": { | |
| "content": "<pr_diff_hunk_comment_line>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110520": { | |
| "content": "<NAME>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110521": { | |
| "content": "<EMAIL>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110522": { | |
| "content": "<KEY>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| }, | |
| "110523": { | |
| "content": "<PASSWORD>", | |
| "lstrip": false, | |
| "normalized": false, | |
| "rstrip": false, | |
| "single_word": false, | |
| "special": true | |
| } | |
| }, | |
| "additional_special_tokens": [ | |
| "<|endoftext|>", | |
| "<|fim_prefix|>", | |
| "<|fim_middle|>", | |
| "<|fim_suffix|>", | |
| "<|endofprompt|>", | |
| "<|_unuse_missing_100256|>", | |
| "<|_unuse_missing_100261|>", | |
| "<|_unuse_missing_100262|>", | |
| "<|_unuse_missing_100263|>", | |
| "<|_unuse_missing_100264|>", | |
| "<|_unuse_missing_100265|>", | |
| "<|_unuse_missing_100266|>", | |
| "<|_unuse_missing_100267|>", | |
| "<|_unuse_missing_100268|>", | |
| "<|_unuse_missing_100269|>", | |
| "<|_unuse_missing_100270|>", | |
| "<|_unuse_missing_100271|>", | |
| "<|im_start|>", | |
| "<|im_end|>", | |
| "<|stop|>", | |
| "<|endofturn|>", | |
| "<repo_name>", | |
| "<file_sep>", | |
| "<issue_start>", | |
| "<issue_comment>", | |
| "<issue_closed>", | |
| "<jupyter_start>", | |
| "<jupyter_text>", | |
| "<jupyter_code>", | |
| "<jupyter_output>", | |
| "<jupyter_script>", | |
| "<empty_output>", | |
| "<code_to_intermediate>", | |
| "<intermediate_to_code>", | |
| "<pr>", | |
| "<pr_status>", | |
| "<pr_is_merged>", | |
| "<pr_base>", | |
| "<pr_file>", | |
| "<pr_base_code>", | |
| "<pr_diff>", | |
| "<pr_diff_hunk>", | |
| "<pr_comment>", | |
| "<pr_event_id>", | |
| "<pr_review>", | |
| "<pr_review_state>", | |
| "<pr_review_comment>", | |
| "<pr_in_reply_to_review_id>", | |
| "<pr_in_reply_to_comment_id>", | |
| "<pr_diff_hunk_comment_line>", | |
| "<NAME>", | |
| "<EMAIL>", | |
| "<KEY>", | |
| "<PASSWORD>" | |
| ], | |
| "bos_token": "<|endoftext|>", | |
| "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}", | |
| "clean_up_tokenization_spaces": true, | |
| "eos_token": "<|endofturn|>", | |
| "extra_special_tokens": {}, | |
| "model_max_length": 1000000000000000019884624838656, | |
| "pad_token": "<|endoftext|>", | |
| "tokenizer_class": "GPT2Tokenizer", | |
| "unk_token": "<|endoftext|>" | |
| } | |