Upload tokenizer.json with huggingface_hub
Browse files- tokenizer.json +7 -7
tokenizer.json
CHANGED
@@ -7214,21 +7214,21 @@
|
|
7214 |
},
|
7215 |
{
|
7216 |
"id": 128798,
|
7217 |
-
"content": "
|
7218 |
"single_word": false,
|
7219 |
"lstrip": false,
|
7220 |
"rstrip": false,
|
7221 |
-
"normalized":
|
7222 |
-
"special":
|
7223 |
},
|
7224 |
{
|
7225 |
"id": 128799,
|
7226 |
-
"content": "
|
7227 |
"single_word": false,
|
7228 |
"lstrip": false,
|
7229 |
"rstrip": false,
|
7230 |
-
"normalized":
|
7231 |
-
"special":
|
7232 |
},
|
7233 |
{
|
7234 |
"id": 128800,
|
@@ -263171,4 +263171,4 @@
|
|
263171 |
"å±ĭ æªIJ"
|
263172 |
]
|
263173 |
}
|
263174 |
-
}
|
|
|
7214 |
},
|
7215 |
{
|
7216 |
"id": 128798,
|
7217 |
+
"content": "<|place▁holder▁no▁798|>",
|
7218 |
"single_word": false,
|
7219 |
"lstrip": false,
|
7220 |
"rstrip": false,
|
7221 |
+
"normalized": false,
|
7222 |
+
"special": true
|
7223 |
},
|
7224 |
{
|
7225 |
"id": 128799,
|
7226 |
+
"content": "<|place▁holder▁no▁799|>",
|
7227 |
"single_word": false,
|
7228 |
"lstrip": false,
|
7229 |
"rstrip": false,
|
7230 |
+
"normalized": false,
|
7231 |
+
"special": true
|
7232 |
},
|
7233 |
{
|
7234 |
"id": 128800,
|
|
|
263171 |
"å±ĭ æªIJ"
|
263172 |
]
|
263173 |
}
|
263174 |
+
}
|