Commit
•
35d0ceb
1
Parent(s):
3d47df5
tokenizer improved
Browse files- tokenizer.json +4 -2
tokenizer.json
CHANGED
|
@@ -110,7 +110,8 @@
|
|
| 110 |
"pre_tokenizer": {
|
| 111 |
"type": "Metaspace",
|
| 112 |
"replacement": "▁",
|
| 113 |
-
"add_prefix_space": true
|
|
|
|
| 114 |
},
|
| 115 |
"post_processor": {
|
| 116 |
"type": "TemplateProcessing",
|
|
@@ -190,7 +191,8 @@
|
|
| 190 |
"decoder": {
|
| 191 |
"type": "Metaspace",
|
| 192 |
"replacement": "▁",
|
| 193 |
-
"add_prefix_space": true
|
|
|
|
| 194 |
},
|
| 195 |
"model": {
|
| 196 |
"type": "Unigram",
|
|
|
|
| 110 |
"pre_tokenizer": {
|
| 111 |
"type": "Metaspace",
|
| 112 |
"replacement": "▁",
|
| 113 |
+
"add_prefix_space": true,
|
| 114 |
+
"prepend_scheme": "always"
|
| 115 |
},
|
| 116 |
"post_processor": {
|
| 117 |
"type": "TemplateProcessing",
|
|
|
|
| 191 |
"decoder": {
|
| 192 |
"type": "Metaspace",
|
| 193 |
"replacement": "▁",
|
| 194 |
+
"add_prefix_space": true,
|
| 195 |
+
"prepend_scheme": "always"
|
| 196 |
},
|
| 197 |
"model": {
|
| 198 |
"type": "Unigram",
|