Upload tokenizer

tokenizer.json

@@ -2,12 +2,14 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 512,
+    "max_length": 256,
     "strategy": "LongestFirst",
     "stride": 0
   },
   "padding": {
-    "strategy": "BatchLongest",
+    "strategy": {
+      "Fixed": 256
+    },
     "direction": "Right",
     "pad_to_multiple_of": null,
     "pad_id": 0,

tokenizer_config.json

@@ -59,7 +59,7 @@
     }
   },
   "bos_token": "<s>",
-  "clean_up_tokenization_spaces": false,
+  "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
   "eos_token": "</s>",
   "errors": "replace",
@@ -67,10 +67,17 @@
   "full_tokenizer_file": null,
   "mask_token": "[MASK]",
   "max_len": 512,
+  "max_length": 512,
   "model_max_length": 512,
+  "pad_to_multiple_of": null,
   "pad_token": "[PAD]",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "sep_token": "[SEP]",
+  "stride": 0,
   "tokenizer_class": "RobertaTokenizer",
   "trim_offsets": true,
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "[UNK]"
 }