opensearch-project
/

opensearch-neural-sparse-encoding-multilingual-v1

passage-retrieval

document-expansion

Model card Files Files and versions Community

zhichao-geng commited on 22 days ago

Commit

4c03a73

·

verified ·

1 Parent(s): 0e4366a

Updated tokenizer config max length

Files changed (1) hide show

tokenizer.json +3 -3

tokenizer.json CHANGED Viewed

@@ -2,7 +2,7 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 256,
     "strategy": "LongestFirst",
     "stride": 0
   },
@@ -1436,7 +1436,7 @@
       "․": 1276,
       "‥": 1277,
       "‧": 1278,
-      " ": 1279,
       "‰": 1280,
       "′": 1281,
       "″": 1282,
@@ -97315,7 +97315,7 @@
       "##․": 97155,
       "##‥": 97156,
       "##‧": 97157,
-      "## ": 97158,
       "##‰": 97159,
       "##′": 97160,
       "##″": 97161,

   "version": "1.0",
   "truncation": {
     "direction": "Right",
+    "max_length": 512,
     "strategy": "LongestFirst",
     "stride": 0
   },
       "․": 1276,
       "‥": 1277,
       "‧": 1278,
       " ": 1279,
       "‰": 1280,
       "′": 1281,
       "″": 1282,
       "##․": 97155,
       "##‥": 97156,
       "##‧": 97157,
       "## ": 97158,
       "##‰": 97159,
       "##′": 97160,
       "##″": 97161,