Commit
•
3151959
1
Parent(s):
65d7de2
tokenizer improved
Browse files- tokenizer.json +3 -1
tokenizer.json
CHANGED
|
@@ -94,6 +94,7 @@
|
|
| 94 |
"pre_tokenizer": {
|
| 95 |
"type": "Metaspace",
|
| 96 |
"replacement": "▁",
|
|
|
|
| 97 |
"prepend_scheme": "never",
|
| 98 |
"split": true
|
| 99 |
},
|
|
@@ -154,6 +155,7 @@
|
|
| 154 |
"decoder": {
|
| 155 |
"type": "Metaspace",
|
| 156 |
"replacement": "▁",
|
|
|
|
| 157 |
"prepend_scheme": "never",
|
| 158 |
"split": true
|
| 159 |
},
|
|
@@ -128164,4 +128166,4 @@
|
|
| 128164 |
],
|
| 128165 |
"byte_fallback": false
|
| 128166 |
}
|
| 128167 |
-
}
|
|
|
|
| 94 |
"pre_tokenizer": {
|
| 95 |
"type": "Metaspace",
|
| 96 |
"replacement": "▁",
|
| 97 |
+
"add_prefix_space": false,
|
| 98 |
"prepend_scheme": "never",
|
| 99 |
"split": true
|
| 100 |
},
|
|
|
|
| 155 |
"decoder": {
|
| 156 |
"type": "Metaspace",
|
| 157 |
"replacement": "▁",
|
| 158 |
+
"add_prefix_space": false,
|
| 159 |
"prepend_scheme": "never",
|
| 160 |
"split": true
|
| 161 |
},
|
|
|
|
| 128166 |
],
|
| 128167 |
"byte_fallback": false
|
| 128168 |
}
|
| 128169 |
+
}
|