Upload tokenizer

added_tokens.json

@@ -0,0 +1,20 @@
+{
+  "<|assistant|>": 32775,
+  "<|available_functions|>": 32770,
+  "<|context|>": 32783,
+  "<|current_states|>": 32782,
+  "<|endoftext|>": 32769,
+  "<|function_call|>": 32776,
+  "<|function_response|>": 32777,
+  "<|gökdeniz|>": 32772,
+  "<|home_state|>": 32781,
+  "<|im_end|>": 32785,
+  "<|im_start|>": 32784,
+  "<|image|>": 32778,
+  "<|josie|>": 32774,
+  "<|long_term_memory|>": 32779,
+  "<|short_term_memory|>": 32780,
+  "<|startoftext|>": 32768,
+  "<|system|>": 32771,
+  "<|user|>": 32773
+}

special_tokens_map.json

@@ -0,0 +1,42 @@
+{
+  "additional_special_tokens": [
+    "<|available_functions|>",
+    "<|system|>",
+    "<|gökdeniz|>",
+    "<|user|>",
+    "<|josie|>",
+    "<|assistant|>",
+    "<|function_call|>",
+    "<|function_response|>",
+    "<|image|>",
+    "<|long_term_memory|>",
+    "<|short_term_memory|>",
+    "<|home_state|>",
+    "<|current_states|>",
+    "<|context|>",
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "[control_768]",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
+size 587404