Upload tokenizer

special_tokens_map.json

@@ -1,7 +1,7 @@
 {
-  "bos_token": "(BOS)",
-  "eos_token": "(EOS)",
-  "mask_token": "(MASK)",
-  "pad_token": "(PAD)",
-  "unk_token": "(UNK)"
+  "bos_token": "([bos])",
+  "eos_token": "([eos])",
+  "mask_token": "([mask])",
+  "pad_token": "([pad])",
+  "unk_token": "([unk])"
 }

tokenizer.json

@@ -5,7 +5,7 @@
   "added_tokens": [
     {
       "id": 0,
-      "content": "(BOS)",
+      "content": "([bos])",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -14,7 +14,7 @@
     },
     {
       "id": 1,
-      "content": "(EOS)",
+      "content": "([eos])",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -23,16 +23,16 @@
     },
     {
       "id": 2,
-      "content": "(UNK)",
+      "content": "([unk])",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": true,
-      "special": false
+      "normalized": false,
+      "special": true
     },
     {
       "id": 3,
-      "content": "(PAD)",
+      "content": "([pad])",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,7 +41,7 @@
     },
     {
       "id": 4,
-      "content": "(MASK)",
+      "content": "([mask])",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -49,7 +49,7 @@
       "special": true
     },
     {
-      "id": 1000,
+      "id": 997,
       "content": "(LNG)",
       "single_word": false,
       "lstrip": false,
@@ -58,7 +58,16 @@
       "special": false
     },
     {
-      "id": 1001,
+      "id": 998,
+      "content": "(UNK)",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 999,
       "content": "(SPN)",
       "single_word": false,
       "lstrip": false,
@@ -85,7 +94,7 @@
       },
       {
         "SpecialToken": {
-          "id": "(EOS)",
+          "id": "([eos])",
           "type_id": 0
         }
       }
@@ -99,7 +108,7 @@
       },
       {
         "SpecialToken": {
-          "id": "(EOS)",
+          "id": "([eos])",
           "type_id": 0
         }
       },
@@ -111,28 +120,28 @@
       },
       {
         "SpecialToken": {
-          "id": "(EOS)",
+          "id": "([eos])",
           "type_id": 1
         }
       }
     ],
     "special_tokens": {
-      "(BOS)": {
-        "id": "(BOS)",
+      "([bos])": {
+        "id": "([bos])",
         "ids": [
           0
         ],
         "tokens": [
-          "(BOS)"
+          "([bos])"
         ]
       },
-      "(EOS)": {
-        "id": "(EOS)",
+      "([eos])": {
+        "id": "([eos])",
         "ids": [
           1
         ],
         "tokens": [
-          "(EOS)"
+          "([eos])"
         ]
       }
     }
@@ -146,17 +155,17 @@
   "model": {
     "type": "BPE",
     "dropout": null,
-    "unk_token": "(UNK)",
+    "unk_token": "([unk])",
     "continuing_subword_prefix": null,
     "end_of_word_suffix": null,
     "fuse_unk": false,
     "byte_fallback": false,
     "vocab": {
-      "(BOS)": 0,
-      "(EOS)": 1,
-      "(UNK)": 2,
-      "(PAD)": 3,
-      "(MASK)": 4,
+      "([bos])": 0,
+      "([eos])": 1,
+      "([unk])": 2,
+      "([pad])": 3,
+      "([mask])": 4,
       "!": 5,
       "%": 6,
       "'": 7,
@@ -1148,10 +1157,7 @@
       "Ġznamen": 993,
       "vý": 994,
       "ovala": 995,
-      "rych": 996,
-      "cká": 997,
-      "ĠJ": 998,
-      "Ġdisku": 999
+      "rych": 996
     },
     "merges": [
       "Ã Ń",
@@ -2011,10 +2017,7 @@
       "Ġzna men",
       "v ý",
       "ova la",
-      "ry ch",
-      "ck á",
-      "Ġ J",
-      "Ġdi sku"
+      "ry ch"
     ]
   }
 }

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "(BOS)",
+      "content": "([bos])",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -9,7 +9,7 @@
       "special": true
     },
     "1": {
-      "content": "(EOS)",
+      "content": "([eos])",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -17,15 +17,15 @@
       "special": true
     },
     "2": {
-      "content": "(UNK)",
+      "content": "([unk])",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "3": {
-      "content": "(PAD)",
+      "content": "([pad])",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -33,14 +33,14 @@
       "special": true
     },
     "4": {
-      "content": "(MASK)",
+      "content": "([mask])",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "1000": {
+    "997": {
       "content": "(LNG)",
       "lstrip": false,
       "normalized": true,
@@ -48,7 +48,15 @@
       "single_word": false,
       "special": false
     },
-    "1001": {
+    "998": {
+      "content": "(UNK)",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "999": {
       "content": "(SPN)",
       "lstrip": false,
       "normalized": true,
@@ -57,12 +65,12 @@
       "special": false
     }
   },
-  "bos_token": "(BOS)",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "(EOS)",
-  "mask_token": "(MASK)",
+  "bos_token": "([bos])",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "([eos])",
+  "mask_token": "([mask])",
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "(PAD)",
+  "pad_token": "([pad])",
   "tokenizer_class": "PreTrainedTokenizerFast",
-  "unk_token": "(UNK)"
+  "unk_token": "([unk])"
 }