kenhktsui commited on
Commit
ca823c2
·
verified ·
1 Parent(s): 129545d

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json CHANGED
@@ -207,6 +207,7 @@
207
  "clean_up_tokenization_spaces": false,
208
  "eos_token": "<|im_end|>",
209
  "errors": "replace",
 
210
  "model_max_length": 32768,
211
  "pad_token": "<|PAD_TOKEN|>",
212
  "padding_side": "left",
 
207
  "clean_up_tokenization_spaces": false,
208
  "eos_token": "<|im_end|>",
209
  "errors": "replace",
210
+ "extra_special_tokens": {},
211
  "model_max_length": 32768,
212
  "pad_token": "<|PAD_TOKEN|>",
213
  "padding_side": "left",