armanc commited on
Commit
7f23112
1 Parent(s): fc0af6e

add tokenizer

Browse files
Files changed (1) hide show
  1. added_tokens.json +1 -1
added_tokens.json CHANGED
@@ -1 +1 @@
1
- {"<reserved-5>": 50270, "<reserved-1>": 50266, "<reserved-7>": 50272, "<reserved-2>": 50267, "<reserved-4>": 50269, "<reserved-9>": 50274, "<reserved-0>": 50265, "<reserved-6>": 50271, "<reserved-8>": 50273, "<reserved-3>": 50268}
 
1
+ {"<reserved-0>": 50265, "<reserved-3>": 50268, "<reserved-5>": 50270, "<reserved-4>": 50269, "<reserved-2>": 50267, "<reserved-8>": 50273, "<reserved-6>": 50271, "<reserved-7>": 50272, "<reserved-1>": 50266, "<reserved-9>": 50274}