Upload tokenizer

special_tokens_map.json

@@ -3,10 +3,7 @@
     "▁<PRE>",
     "▁<MID>",
     "▁<SUF>",
-    "▁<EOT>",
-    ">>>>>>>",
-    "=======",
-    "<<<<<<<"
+    "▁<EOT>"
   ],
   "bos_token": "<s>",
   "eos_token": "</s>",

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 512,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {
@@ -65,33 +70,6 @@
       "rstrip": true,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 32016,
-      "content": ">>>>>>>",
-      "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32017,
-      "content": "=======",
-      "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32018,
-      "content": "<<<<<<<",
-      "single_word": false,
-      "lstrip": true,
-      "rstrip": true,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -55,40 +55,13 @@
       "rstrip": true,
       "single_word": false,
       "special": true
-    },
-    "32016": {
-      "content": ">>>>>>>",
-      "lstrip": true,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32017": {
-      "content": "=======",
-      "lstrip": true,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "32018": {
-      "content": "<<<<<<<",
-      "lstrip": true,
-      "normalized": false,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
     }
   },
   "additional_special_tokens": [
     "▁<PRE>",
     "▁<MID>",
     "▁<SUF>",
-    "▁<EOT>",
-    ">>>>>>>",
-    "=======",
-    "<<<<<<<"
+    "▁<EOT>"
   ],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,