Upload tokenizer

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|inst_end|>": 50258,
+  "<|inst_start|>": 50257
+}

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|inst_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,41 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50257": {
+      "content": "<|inst_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50258": {
+      "content": "<|inst_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|inst_end|>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "chat_template": "\n    {{- bos_token }} \n    {%- for message in messages %}\n        {%- if message['role'] != 'assistant' %}\n            {{- '<|inst_start|> ' + message['content'] + ' <|inst_end|> ' }}\n        {%- else %}\n            {{- message['content'] }}\n        {%- endif %}\n    {%- endfor %}\n    {{- eos_token }}\n    ",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}