Upload tokenizer Upload version v18.0 (Trained with Unsloth)

added_tokens.json

@@ -1,3 +1,40 @@
 {
-  "<image_soft_token>": 262144
+  "#Address#": 151672,
+  "#CarNumber#": 151673,
+  "#CardNumber#": 151674,
+  "#DateOfBirth#": 151675,
+  "#Email#": 151676,
+  "#PassportNumber#": 151677,
+  "#Person#": 151678,
+  "#Person1#": 151665,
+  "#Person2#": 151666,
+  "#Person3#": 151667,
+  "#Person4#": 151668,
+  "#Person5#": 151669,
+  "#Person6#": 151670,
+  "#Person7#": 151671,
+  "#PhoneNumber#": 151679,
+  "#SSN#": 151680,
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
 }

special_tokens_map.json

@@ -1,30 +1,28 @@
 {
-  "boi_token": "<start_of_image>",
-  "bos_token": {
-    "content": "<bos>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eoi_token": "<end_of_image>",
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
   "eos_token": {
-    "content": "<end_of_turn>",
+    "content": "<|endoftext|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "image_token": "<image_soft_token>",
   "pad_token": {
-    "content": "<pad>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
+    "content": "<|vision_pad|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
-size 33384568
+oid sha256:aab041e54bd9d8f05e954a5eeea0c1619f88b2390ed097dd02f2da81b8f5cbe7
+size 11424885