Upload tokenizer

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,672 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 256,
+      "content": "<|begin_of_text|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 265,
+      "content": "<|eot_id|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 512,
+      "content": "ĊĊ",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 513,
+      "content": "user",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 514,
+      "content": "assistant",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 515,
+      "content": "system",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      },
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": false,
+        "trim_offsets": true,
+        "use_regex": false
+      }
+    ]
+  },
+  "post_processor": {
+    "type": "Sequence",
+    "processors": [
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": true,
+        "trim_offsets": false,
+        "use_regex": true
+      },
+      {
+        "type": "TemplateProcessing",
+        "single": [
+          {
+            "SpecialToken": {
+              "id": "<|begin_of_text|>",
+              "type_id": 0
+            }
+          },
+          {
+            "Sequence": {
+              "id": "A",
+              "type_id": 0
+            }
+          }
+        ],
+        "pair": [
+          {
+            "SpecialToken": {
+              "id": "<|begin_of_text|>",
+              "type_id": 0
+            }
+          },
+          {
+            "Sequence": {
+              "id": "A",
+              "type_id": 0
+            }
+          },
+          {
+            "SpecialToken": {
+              "id": "<|begin_of_text|>",
+              "type_id": 1
+            }
+          },
+          {
+            "Sequence": {
+              "id": "B",
+              "type_id": 1
+            }
+          }
+        ],
+        "special_tokens": {
+          "<|begin_of_text|>": {
+            "id": "<|begin_of_text|>",
+            "ids": [
+              256
+            ],
+            "tokens": [
+              "<|begin_of_text|>"
+            ]
+          }
+        }
+      }
+    ]
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "WordPiece",
+    "unk_token": "<|eot_id|>",
+    "continuing_subword_prefix": "",
+    "max_input_chars_per_word": 1000000,
+    "vocab": {
+      "Ā": 0,
+      "ā": 1,
+      "Ă": 2,
+      "ă": 3,
+      "Ą": 4,
+      "ą": 5,
+      "Ć": 6,
+      "ć": 7,
+      "Ĉ": 8,
+      "ĉ": 9,
+      "Ċ": 10,
+      "ċ": 11,
+      "Č": 12,
+      "č": 13,
+      "Ď": 14,
+      "ď": 15,
+      "Đ": 16,
+      "đ": 17,
+      "Ē": 18,
+      "ē": 19,
+      "Ĕ": 20,
+      "ĕ": 21,
+      "Ė": 22,
+      "ė": 23,
+      "Ę": 24,
+      "ę": 25,
+      "Ě": 26,
+      "ě": 27,
+      "Ĝ": 28,
+      "ĝ": 29,
+      "Ğ": 30,
+      "ğ": 31,
+      "Ġ": 32,
+      "!": 33,
+      "\"": 34,
+      "#": 35,
+      "$": 36,
+      "%": 37,
+      "&": 38,
+      "'": 39,
+      "(": 40,
+      ")": 41,
+      "*": 42,
+      "+": 43,
+      ",": 44,
+      "-": 45,
+      ".": 46,
+      "/": 47,
+      "0": 48,
+      "1": 49,
+      "2": 50,
+      "3": 51,
+      "4": 52,
+      "5": 53,
+      "6": 54,
+      "7": 55,
+      "8": 56,
+      "9": 57,
+      ":": 58,
+      ";": 59,
+      "<": 60,
+      "=": 61,
+      ">": 62,
+      "?": 63,
+      "@": 64,
+      "A": 65,
+      "B": 66,
+      "C": 67,
+      "D": 68,
+      "E": 69,
+      "F": 70,
+      "G": 71,
+      "H": 72,
+      "I": 73,
+      "J": 74,
+      "K": 75,
+      "L": 76,
+      "M": 77,
+      "N": 78,
+      "O": 79,
+      "P": 80,
+      "Q": 81,
+      "R": 82,
+      "S": 83,
+      "T": 84,
+      "U": 85,
+      "V": 86,
+      "W": 87,
+      "X": 88,
+      "Y": 89,
+      "Z": 90,
+      "[": 91,
+      "\\": 92,
+      "]": 93,
+      "^": 94,
+      "_": 95,
+      "`": 96,
+      "a": 97,
+      "b": 98,
+      "c": 99,
+      "d": 100,
+      "e": 101,
+      "f": 102,
+      "g": 103,
+      "h": 104,
+      "i": 105,
+      "j": 106,
+      "k": 107,
+      "l": 108,
+      "m": 109,
+      "n": 110,
+      "o": 111,
+      "p": 112,
+      "q": 113,
+      "r": 114,
+      "s": 115,
+      "t": 116,
+      "u": 117,
+      "v": 118,
+      "w": 119,
+      "x": 120,
+      "y": 121,
+      "z": 122,
+      "{": 123,
+      "|": 124,
+      "}": 125,
+      "~": 126,
+      "ġ": 127,
+      "Ģ": 128,
+      "ģ": 129,
+      "Ĥ": 130,
+      "ĥ": 131,
+      "Ħ": 132,
+      "ħ": 133,
+      "Ĩ": 134,
+      "ĩ": 135,
+      "Ī": 136,
+      "ī": 137,
+      "Ĭ": 138,
+      "ĭ": 139,
+      "Į": 140,
+      "į": 141,
+      "İ": 142,
+      "ı": 143,
+      "IJ": 144,
+      "ij": 145,
+      "Ĵ": 146,
+      "ĵ": 147,
+      "Ķ": 148,
+      "ķ": 149,
+      "ĸ": 150,
+      "Ĺ": 151,
+      "ĺ": 152,
+      "Ļ": 153,
+      "ļ": 154,
+      "Ľ": 155,
+      "ľ": 156,
+      "Ŀ": 157,
+      "ŀ": 158,
+      "Ł": 159,
+      "ł": 160,
+      "¡": 161,
+      "¢": 162,
+      "£": 163,
+      "¤": 164,
+      "¥": 165,
+      "¦": 166,
+      "§": 167,
+      "¨": 168,
+      "©": 169,
+      "ª": 170,
+      "«": 171,
+      "¬": 172,
+      "Ń": 173,
+      "®": 174,
+      "¯": 175,
+      "°": 176,
+      "±": 177,
+      "²": 178,
+      "³": 179,
+      "´": 180,
+      "µ": 181,
+      "¶": 182,
+      "·": 183,
+      "¸": 184,
+      "¹": 185,
+      "º": 186,
+      "»": 187,
+      "¼": 188,
+      "½": 189,
+      "¾": 190,
+      "¿": 191,
+      "À": 192,
+      "Á": 193,
+      "Â": 194,
+      "Ã": 195,
+      "Ä": 196,
+      "Å": 197,
+      "Æ": 198,
+      "Ç": 199,
+      "È": 200,
+      "É": 201,
+      "Ê": 202,
+      "Ë": 203,
+      "Ì": 204,
+      "Í": 205,
+      "Î": 206,
+      "Ï": 207,
+      "Ð": 208,
+      "Ñ": 209,
+      "Ò": 210,
+      "Ó": 211,
+      "Ô": 212,
+      "Õ": 213,
+      "Ö": 214,
+      "×": 215,
+      "Ø": 216,
+      "Ù": 217,
+      "Ú": 218,
+      "Û": 219,
+      "Ü": 220,
+      "Ý": 221,
+      "Þ": 222,
+      "ß": 223,
+      "à": 224,
+      "á": 225,
+      "â": 226,
+      "ã": 227,
+      "ä": 228,
+      "å": 229,
+      "æ": 230,
+      "ç": 231,
+      "è": 232,
+      "é": 233,
+      "ê": 234,
+      "ë": 235,
+      "ì": 236,
+      "í": 237,
+      "î": 238,
+      "ï": 239,
+      "ð": 240,
+      "ñ": 241,
+      "ò": 242,
+      "ó": 243,
+      "ô": 244,
+      "õ": 245,
+      "ö": 246,
+      "÷": 247,
+      "ø": 248,
+      "ù": 249,
+      "ú": 250,
+      "û": 251,
+      "ü": 252,
+      "ý": 253,
+      "þ": 254,
+      "ÿ": 255,
+      "<|begin_of_text|>": 256,
+      "<|end_of_text|>": 257,
+      "<|reserved_special_token_0|>": 258,
+      "<|reserved_special_token_1|>": 259,
+      "<|finetune_right_pad_id|>": 260,
+      "<|reserved_special_token_2|>": 261,
+      "<|start_header_id|>": 262,
+      "<|end_header_id|>": 263,
+      "<|eom_id|>": 264,
+      "<|eot_id|>": 265,
+      "<|python_tag|>": 266,
+      "<|reserved_special_token_3|>": 267,
+      "<|reserved_special_token_4|>": 268,
+      "<|reserved_special_token_5|>": 269,
+      "<|reserved_special_token_6|>": 270,
+      "<|reserved_special_token_7|>": 271,
+      "<|reserved_special_token_8|>": 272,
+      "<|reserved_special_token_9|>": 273,
+      "<|reserved_special_token_10|>": 274,
+      "<|reserved_special_token_11|>": 275,
+      "<|reserved_special_token_12|>": 276,
+      "<|reserved_special_token_13|>": 277,
+      "<|reserved_special_token_14|>": 278,
+      "<|reserved_special_token_15|>": 279,
+      "<|reserved_special_token_16|>": 280,
+      "<|reserved_special_token_17|>": 281,
+      "<|reserved_special_token_18|>": 282,
+      "<|reserved_special_token_19|>": 283,
+      "<|reserved_special_token_20|>": 284,
+      "<|reserved_special_token_21|>": 285,
+      "<|reserved_special_token_22|>": 286,
+      "<|reserved_special_token_23|>": 287,
+      "<|reserved_special_token_24|>": 288,
+      "<|reserved_special_token_25|>": 289,
+      "<|reserved_special_token_26|>": 290,
+      "<|reserved_special_token_27|>": 291,
+      "<|reserved_special_token_28|>": 292,
+      "<|reserved_special_token_29|>": 293,
+      "<|reserved_special_token_30|>": 294,
+      "<|reserved_special_token_31|>": 295,
+      "<|reserved_special_token_32|>": 296,
+      "<|reserved_special_token_33|>": 297,
+      "<|reserved_special_token_34|>": 298,
+      "<|reserved_special_token_35|>": 299,
+      "<|reserved_special_token_36|>": 300,
+      "<|reserved_special_token_37|>": 301,
+      "<|reserved_special_token_38|>": 302,
+      "<|reserved_special_token_39|>": 303,
+      "<|reserved_special_token_40|>": 304,
+      "<|reserved_special_token_41|>": 305,
+      "<|reserved_special_token_42|>": 306,
+      "<|reserved_special_token_43|>": 307,
+      "<|reserved_special_token_44|>": 308,
+      "<|reserved_special_token_45|>": 309,
+      "<|reserved_special_token_46|>": 310,
+      "<|reserved_special_token_47|>": 311,
+      "<|reserved_special_token_48|>": 312,
+      "<|reserved_special_token_49|>": 313,
+      "<|reserved_special_token_50|>": 314,
+      "<|reserved_special_token_51|>": 315,
+      "<|reserved_special_token_52|>": 316,
+      "<|reserved_special_token_53|>": 317,
+      "<|reserved_special_token_54|>": 318,
+      "<|reserved_special_token_55|>": 319,
+      "<|reserved_special_token_56|>": 320,
+      "<|reserved_special_token_57|>": 321,
+      "<|reserved_special_token_58|>": 322,
+      "<|reserved_special_token_59|>": 323,
+      "<|reserved_special_token_60|>": 324,
+      "<|reserved_special_token_61|>": 325,
+      "<|reserved_special_token_62|>": 326,
+      "<|reserved_special_token_63|>": 327,
+      "<|reserved_special_token_64|>": 328,
+      "<|reserved_special_token_65|>": 329,
+      "<|reserved_special_token_66|>": 330,
+      "<|reserved_special_token_67|>": 331,
+      "<|reserved_special_token_68|>": 332,
+      "<|reserved_special_token_69|>": 333,
+      "<|reserved_special_token_70|>": 334,
+      "<|reserved_special_token_71|>": 335,
+      "<|reserved_special_token_72|>": 336,
+      "<|reserved_special_token_73|>": 337,
+      "<|reserved_special_token_74|>": 338,
+      "<|reserved_special_token_75|>": 339,
+      "<|reserved_special_token_76|>": 340,
+      "<|reserved_special_token_77|>": 341,
+      "<|reserved_special_token_78|>": 342,
+      "<|reserved_special_token_79|>": 343,
+      "<|reserved_special_token_80|>": 344,
+      "<|reserved_special_token_81|>": 345,
+      "<|reserved_special_token_82|>": 346,
+      "<|reserved_special_token_83|>": 347,
+      "<|reserved_special_token_84|>": 348,
+      "<|reserved_special_token_85|>": 349,
+      "<|reserved_special_token_86|>": 350,
+      "<|reserved_special_token_87|>": 351,
+      "<|reserved_special_token_88|>": 352,
+      "<|reserved_special_token_89|>": 353,
+      "<|reserved_special_token_90|>": 354,
+      "<|reserved_special_token_91|>": 355,
+      "<|reserved_special_token_92|>": 356,
+      "<|reserved_special_token_93|>": 357,
+      "<|reserved_special_token_94|>": 358,
+      "<|reserved_special_token_95|>": 359,
+      "<|reserved_special_token_96|>": 360,
+      "<|reserved_special_token_97|>": 361,
+      "<|reserved_special_token_98|>": 362,
+      "<|reserved_special_token_99|>": 363,
+      "<|reserved_special_token_100|>": 364,
+      "<|reserved_special_token_101|>": 365,
+      "<|reserved_special_token_102|>": 366,
+      "<|reserved_special_token_103|>": 367,
+      "<|reserved_special_token_104|>": 368,
+      "<|reserved_special_token_105|>": 369,
+      "<|reserved_special_token_106|>": 370,
+      "<|reserved_special_token_107|>": 371,
+      "<|reserved_special_token_108|>": 372,
+      "<|reserved_special_token_109|>": 373,
+      "<|reserved_special_token_110|>": 374,
+      "<|reserved_special_token_111|>": 375,
+      "<|reserved_special_token_112|>": 376,
+      "<|reserved_special_token_113|>": 377,
+      "<|reserved_special_token_114|>": 378,
+      "<|reserved_special_token_115|>": 379,
+      "<|reserved_special_token_116|>": 380,
+      "<|reserved_special_token_117|>": 381,
+      "<|reserved_special_token_118|>": 382,
+      "<|reserved_special_token_119|>": 383,
+      "<|reserved_special_token_120|>": 384,
+      "<|reserved_special_token_121|>": 385,
+      "<|reserved_special_token_122|>": 386,
+      "<|reserved_special_token_123|>": 387,
+      "<|reserved_special_token_124|>": 388,
+      "<|reserved_special_token_125|>": 389,
+      "<|reserved_special_token_126|>": 390,
+      "<|reserved_special_token_127|>": 391,
+      "<|reserved_special_token_128|>": 392,
+      "<|reserved_special_token_129|>": 393,
+      "<|reserved_special_token_130|>": 394,
+      "<|reserved_special_token_131|>": 395,
+      "<|reserved_special_token_132|>": 396,
+      "<|reserved_special_token_133|>": 397,
+      "<|reserved_special_token_134|>": 398,
+      "<|reserved_special_token_135|>": 399,
+      "<|reserved_special_token_136|>": 400,
+      "<|reserved_special_token_137|>": 401,
+      "<|reserved_special_token_138|>": 402,
+      "<|reserved_special_token_139|>": 403,
+      "<|reserved_special_token_140|>": 404,
+      "<|reserved_special_token_141|>": 405,
+      "<|reserved_special_token_142|>": 406,
+      "<|reserved_special_token_143|>": 407,
+      "<|reserved_special_token_144|>": 408,
+      "<|reserved_special_token_145|>": 409,
+      "<|reserved_special_token_146|>": 410,
+      "<|reserved_special_token_147|>": 411,
+      "<|reserved_special_token_148|>": 412,
+      "<|reserved_special_token_149|>": 413,
+      "<|reserved_special_token_150|>": 414,
+      "<|reserved_special_token_151|>": 415,
+      "<|reserved_special_token_152|>": 416,
+      "<|reserved_special_token_153|>": 417,
+      "<|reserved_special_token_154|>": 418,
+      "<|reserved_special_token_155|>": 419,
+      "<|reserved_special_token_156|>": 420,
+      "<|reserved_special_token_157|>": 421,
+      "<|reserved_special_token_158|>": 422,
+      "<|reserved_special_token_159|>": 423,
+      "<|reserved_special_token_160|>": 424,
+      "<|reserved_special_token_161|>": 425,
+      "<|reserved_special_token_162|>": 426,
+      "<|reserved_special_token_163|>": 427,
+      "<|reserved_special_token_164|>": 428,
+      "<|reserved_special_token_165|>": 429,
+      "<|reserved_special_token_166|>": 430,
+      "<|reserved_special_token_167|>": 431,
+      "<|reserved_special_token_168|>": 432,
+      "<|reserved_special_token_169|>": 433,
+      "<|reserved_special_token_170|>": 434,
+      "<|reserved_special_token_171|>": 435,
+      "<|reserved_special_token_172|>": 436,
+      "<|reserved_special_token_173|>": 437,
+      "<|reserved_special_token_174|>": 438,
+      "<|reserved_special_token_175|>": 439,
+      "<|reserved_special_token_176|>": 440,
+      "<|reserved_special_token_177|>": 441,
+      "<|reserved_special_token_178|>": 442,
+      "<|reserved_special_token_179|>": 443,
+      "<|reserved_special_token_180|>": 444,
+      "<|reserved_special_token_181|>": 445,
+      "<|reserved_special_token_182|>": 446,
+      "<|reserved_special_token_183|>": 447,
+      "<|reserved_special_token_184|>": 448,
+      "<|reserved_special_token_185|>": 449,
+      "<|reserved_special_token_186|>": 450,
+      "<|reserved_special_token_187|>": 451,
+      "<|reserved_special_token_188|>": 452,
+      "<|reserved_special_token_189|>": 453,
+      "<|reserved_special_token_190|>": 454,
+      "<|reserved_special_token_191|>": 455,
+      "<|reserved_special_token_192|>": 456,
+      "<|reserved_special_token_193|>": 457,
+      "<|reserved_special_token_194|>": 458,
+      "<|reserved_special_token_195|>": 459,
+      "<|reserved_special_token_196|>": 460,
+      "<|reserved_special_token_197|>": 461,
+      "<|reserved_special_token_198|>": 462,
+      "<|reserved_special_token_199|>": 463,
+      "<|reserved_special_token_200|>": 464,
+      "<|reserved_special_token_201|>": 465,
+      "<|reserved_special_token_202|>": 466,
+      "<|reserved_special_token_203|>": 467,
+      "<|reserved_special_token_204|>": 468,
+      "<|reserved_special_token_205|>": 469,
+      "<|reserved_special_token_206|>": 470,
+      "<|reserved_special_token_207|>": 471,
+      "<|reserved_special_token_208|>": 472,
+      "<|reserved_special_token_209|>": 473,
+      "<|reserved_special_token_210|>": 474,
+      "<|reserved_special_token_211|>": 475,
+      "<|reserved_special_token_212|>": 476,
+      "<|reserved_special_token_213|>": 477,
+      "<|reserved_special_token_214|>": 478,
+      "<|reserved_special_token_215|>": 479,
+      "<|reserved_special_token_216|>": 480,
+      "<|reserved_special_token_217|>": 481,
+      "<|reserved_special_token_218|>": 482,
+      "<|reserved_special_token_219|>": 483,
+      "<|reserved_special_token_220|>": 484,
+      "<|reserved_special_token_221|>": 485,
+      "<|reserved_special_token_222|>": 486,
+      "<|reserved_special_token_223|>": 487,
+      "<|reserved_special_token_224|>": 488,
+      "<|reserved_special_token_225|>": 489,
+      "<|reserved_special_token_226|>": 490,
+      "<|reserved_special_token_227|>": 491,
+      "<|reserved_special_token_228|>": 492,
+      "<|reserved_special_token_229|>": 493,
+      "<|reserved_special_token_230|>": 494,
+      "<|reserved_special_token_231|>": 495,
+      "<|reserved_special_token_232|>": 496,
+      "<|reserved_special_token_233|>": 497,
+      "<|reserved_special_token_234|>": 498,
+      "<|reserved_special_token_235|>": 499,
+      "<|reserved_special_token_236|>": 500,
+      "<|reserved_special_token_237|>": 501,
+      "<|reserved_special_token_238|>": 502,
+      "<|reserved_special_token_239|>": 503,
+      "<|reserved_special_token_240|>": 504,
+      "<|reserved_special_token_241|>": 505,
+      "<|reserved_special_token_242|>": 506,
+      "<|reserved_special_token_243|>": 507,
+      "<|reserved_special_token_244|>": 508,
+      "<|reserved_special_token_245|>": 509,
+      "<|reserved_special_token_246|>": 510,
+      "<|reserved_special_token_247|>": 511
+    }
+  }
+}

tokenizer_config.json

@@ -0,0 +1,63 @@
+{
+  "added_tokens_decoder": {
+    "256": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "265": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "512": {
+      "content": "ĊĊ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "513": {
+      "content": "user",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "514": {
+      "content": "assistant",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "515": {
+      "content": "system",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now(\"%d %b %Y\") %}\n    {%- else %}\n        {%- set date_string = \"26 Jul 2024\" %}\n    {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n        {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n        {{- '\"parameters\": ' }}\n        {{- tool_call.arguments | tojson }}\n        {{- \"}\" }}\n        {{- \"<|eot_id|>\" }}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}