Upload folder using huggingface_hub

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c1e468ff18a8a93027bf970cc9267c6ee62d2719edbc5329d9be198d98fb674
+oid sha256:0fc0df86988365766bd8ccb29f2cfaf1d4efaefd7a909f32161a270b2f280d9d
 size 2390960584

special_tokens_map.json

@@ -2,18 +2,22 @@
   "additional_special_tokens": [
     "<|query_start|>",
     "<|query_end|>",
+    "<|source_start|>",
     "<|source_id|>",
+    "<|source_end|>",
+    "<|language_start|>",
+    "<|language_end|>",
     "<|query_analysis_start|>",
     "<|query_analysis_end|>",
     "<|query_report_start|>",
     "<|query_report_end|>",
     "<|source_analysis_start|>",
     "<|source_analysis_end|>",
+    "<|source_report_start|>",
+    "<|source_report_end|>",
     "<|draft_start|>",
     "<|draft_end|>",
     "<|answer_start|>",
-    "<|answer_end|>",
-    "<|source_start|>",
-    "<|source_end|>"
+    "<|answer_end|>"
   ]
 }

tokenizer.json

@@ -40,7 +40,7 @@
       "special": true
     },
     {
-      "id": 65521,
+      "id": 65517,
       "content": "<|query_start|>",
       "single_word": false,
       "lstrip": false,
@@ -49,7 +49,7 @@
       "special": true
     },
     {
-      "id": 65522,
+      "id": 65518,
       "content": "<|query_end|>",
       "single_word": false,
       "lstrip": false,
@@ -58,7 +58,16 @@
       "special": true
     },
     {
-      "id": 65523,
+      "id": 65519,
+      "content": "<|source_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65520,
       "content": "<|source_id|>",
       "single_word": false,
       "lstrip": false,
@@ -66,6 +75,33 @@
       "normalized": false,
       "special": true
     },
+    {
+      "id": 65521,
+      "content": "<|source_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65522,
+      "content": "<|language_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65523,
+      "content": "<|language_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
     {
       "id": 65524,
       "content": "<|query_analysis_start|>",
@@ -122,7 +158,7 @@
     },
     {
       "id": 65530,
-      "content": "<|draft_start|>",
+      "content": "<|source_report_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -131,7 +167,7 @@
     },
     {
       "id": 65531,
-      "content": "<|draft_end|>",
+      "content": "<|source_report_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -140,7 +176,7 @@
     },
     {
       "id": 65532,
-      "content": "<|answer_start|>",
+      "content": "<|draft_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -149,7 +185,7 @@
     },
     {
       "id": 65533,
-      "content": "<|answer_end|>",
+      "content": "<|draft_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -158,7 +194,7 @@
     },
     {
       "id": 65534,
-      "content": "<|source_start|>",
+      "content": "<|answer_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -167,7 +203,7 @@
     },
     {
       "id": 65535,
-      "content": "<|source_end|>",
+      "content": "<|answer_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -195,7 +231,98 @@
       }
     ]
   },
-  "post_processor": null,
+  "post_processor": {
+    "type": "Sequence",
+    "processors": [
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": true,
+        "trim_offsets": false,
+        "use_regex": true
+      },
+      {
+        "type": "TemplateProcessing",
+        "single": [
+          {
+            "SpecialToken": {
+              "id": "<|end_of_text|>",
+              "type_id": 0
+            }
+          },
+          {
+            "Sequence": {
+              "id": "A",
+              "type_id": 0
+            }
+          }
+        ],
+        "pair": [
+          {
+            "SpecialToken": {
+              "id": "<|end_of_text|>",
+              "type_id": 0
+            }
+          },
+          {
+            "Sequence": {
+              "id": "A",
+              "type_id": 0
+            }
+          },
+          {
+            "SpecialToken": {
+              "id": "<|end_of_text|>",
+              "type_id": 0
+            }
+          },
+          {
+            "Sequence": {
+              "id": "B",
+              "type_id": 0
+            }
+          }
+        ],
+        "special_tokens": {
+          "<|begin_of_text|>": {
+            "id": "<|begin_of_text|>",
+            "ids": [
+              1
+            ],
+            "tokens": [
+              "<|begin_of_text|>"
+            ]
+          },
+          "<|end_of_text|>": {
+            "id": "<|end_of_text|>",
+            "ids": [
+              2
+            ],
+            "tokens": [
+              "<|end_of_text|>"
+            ]
+          },
+          "[PAD]": {
+            "id": "[PAD]",
+            "ids": [
+              3
+            ],
+            "tokens": [
+              "[PAD]"
+            ]
+          },
+          "[UNK]": {
+            "id": "[UNK]",
+            "ids": [
+              0
+            ],
+            "tokens": [
+              "[UNK]"
+            ]
+          }
+        }
+      }
+    ]
+  },
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,
@@ -65728,11 +65855,7 @@
       "Ġ709": 65513,
       "Ġménages": 65514,
       "Ġprodukty": 65515,
-      "Ġsuum": 65516,
-      "Ġinfluenc": 65517,
-      "Ġcatching": 65518,
-      "ĠÏĮÏĦαν": 65519,
-      "737": 65520
+      "Ġsuum": 65516
     },
     "merges": [
       "Ġ Ġ",
@@ -131004,11 +131127,7 @@
       "Ġ7 09",
       "Ġmén ages",
       "Ġproduk ty",
-      "Ġsu um",
-      "Ġinflu enc",
-      "Ġcatch ing",
-      "ĠÏĮ ÏĦαν",
-      "7 37"
+      "Ġsu um"
     ]
   }
 }

tokenizer_config.json

@@ -32,7 +32,7 @@
       "single_word": false,
       "special": true
     },
-    "65521": {
+    "65517": {
       "content": "<|query_start|>",
       "lstrip": false,
       "normalized": false,
@@ -40,7 +40,7 @@
       "single_word": false,
       "special": true
     },
-    "65522": {
+    "65518": {
       "content": "<|query_end|>",
       "lstrip": false,
       "normalized": false,
@@ -48,7 +48,15 @@
       "single_word": false,
       "special": true
     },
-    "65523": {
+    "65519": {
+      "content": "<|source_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65520": {
       "content": "<|source_id|>",
       "lstrip": false,
       "normalized": false,
@@ -56,6 +64,30 @@
       "single_word": false,
       "special": true
     },
+    "65521": {
+      "content": "<|source_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65522": {
+      "content": "<|language_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65523": {
+      "content": "<|language_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
     "65524": {
       "content": "<|query_analysis_start|>",
       "lstrip": false,
@@ -105,7 +137,7 @@
       "special": true
     },
     "65530": {
-      "content": "<|draft_start|>",
+      "content": "<|source_report_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -113,7 +145,7 @@
       "special": true
     },
     "65531": {
-      "content": "<|draft_end|>",
+      "content": "<|source_report_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -121,7 +153,7 @@
       "special": true
     },
     "65532": {
-      "content": "<|answer_start|>",
+      "content": "<|draft_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -129,7 +161,7 @@
       "special": true
     },
     "65533": {
-      "content": "<|answer_end|>",
+      "content": "<|draft_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -137,7 +169,7 @@
       "special": true
     },
     "65534": {
-      "content": "<|source_start|>",
+      "content": "<|answer_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -145,7 +177,7 @@
       "special": true
     },
     "65535": {
-      "content": "<|source_end|>",
+      "content": "<|answer_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -156,19 +188,23 @@
   "additional_special_tokens": [
     "<|query_start|>",
     "<|query_end|>",
+    "<|source_start|>",
     "<|source_id|>",
+    "<|source_end|>",
+    "<|language_start|>",
+    "<|language_end|>",
     "<|query_analysis_start|>",
     "<|query_analysis_end|>",
     "<|query_report_start|>",
     "<|query_report_end|>",
     "<|source_analysis_start|>",
     "<|source_analysis_end|>",
+    "<|source_report_start|>",
+    "<|source_report_end|>",
     "<|draft_start|>",
     "<|draft_end|>",
     "<|answer_start|>",
-    "<|answer_end|>",
-    "<|source_start|>",
-    "<|source_end|>"
+    "<|answer_end|>"
   ],
   "clean_up_tokenization_spaces": true,
   "model_max_length": 1000000000000000019884624838656,