bilalfaye commited on
Commit
4867803
·
verified ·
1 Parent(s): 7f6db95

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -30,7 +30,7 @@
30
  "<|inst_end|>"
31
  ],
32
  "bos_token": "<|endoftext|>",
33
- "chat_template": "\n {{- bos_token }} \n {%- for message in messages %}\n {%- if message['role'] != 'assistant' %}\n {{- '<|inst_start|> ' + message['content'] + ' <|inst_end|> ' }}\n {%- else %}\n {{- message['content'] }}\n {%- endif %}\n {%- endfor %}\n {{- eos_token }}\n ",
34
  "clean_up_tokenization_spaces": false,
35
  "eos_token": "<|endoftext|>",
36
  "extra_special_tokens": {},
 
30
  "<|inst_end|>"
31
  ],
32
  "bos_token": "<|endoftext|>",
33
+ "chat_template": "{{ bos_token }}{% for message in messages %}{% if message['role'] != 'assistant' %}<|inst_start|>{{ message['content'] }}<|inst_end|>{% else %}{{ message['content'] }}{% endif %}{% endfor %}{{ eos_token }}",
34
  "clean_up_tokenization_spaces": false,
35
  "eos_token": "<|endoftext|>",
36
  "extra_special_tokens": {},