Upload tokenizer

added_tokens.json

@@ -0,0 +1,11 @@
+{
+  "<|endanswer|>": 32769,
+  "<|enddocument|>": 32771,
+  "<|endpos|>": 32776,
+  "<|endtask|>": 32773,
+  "<|pad|>": 32770,
+  "<|startanswer|>": 32774,
+  "<|startdocument|>": 32768,
+  "<|startpos|>": 32775,
+  "<|starttask|>": 32772
+}

special_tokens_map.json

@@ -0,0 +1,41 @@
+{
+  "additional_special_tokens": [
+    "<|endanswer|>",
+    "<|pad|>",
+    "<|startdocument|>",
+    "<|enddocument|>",
+    "<|starttask|>",
+    "<|endtask|>",
+    "<|startanswer|>",
+    "<|startpos|>",
+    "<|endpos|>"
+  ],
+  "bos_token": {
+    "content": "<|startdocument|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endanswer|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
+size 587404