hamishivi commited on
Commit
b7fe705
·
verified ·
1 Parent(s): 4836fb3

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -9
tokenizer_config.json CHANGED
@@ -2041,21 +2041,13 @@
2041
  "special": true
2042
  },
2043
  "128255": {
2044
- "content": "<|reserved_special_token_250|>",
2045
  "lstrip": false,
2046
  "normalized": false,
2047
  "rstrip": false,
2048
  "single_word": false,
2049
  "special": true
2050
  },
2051
- "128256": {
2052
- "content": "<pad>",
2053
- "lstrip": false,
2054
- "normalized": false,
2055
- "rstrip": false,
2056
- "single_word": false,
2057
- "special": true
2058
- }
2059
  },
2060
  "bos_token": "<|begin_of_text|>",
2061
  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
 
2041
  "special": true
2042
  },
2043
  "128255": {
2044
+ "content": "<padd>",
2045
  "lstrip": false,
2046
  "normalized": false,
2047
  "rstrip": false,
2048
  "single_word": false,
2049
  "special": true
2050
  },
 
 
 
 
 
 
 
 
2051
  },
2052
  "bos_token": "<|begin_of_text|>",
2053
  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",