opensearch-project
/

opensearch-neural-sparse-encoding-v2-distill

Feature Extraction

sentence-transformers

passage-retrieval

query-expansion

document-expansion

text-embeddings-inference

Model card Files Files and versions

Tom Aarsen commited on Apr 17

Commit

bd901a6

·

1 Parent(s): 8af4c0e

Use the DistilBertTokenizer for this DistilBERT-based model

Files changed (2) hide show

README.md +1 -1
tokenizer_config.json +1 -1

README.md CHANGED Viewed

@@ -88,7 +88,7 @@ query = "What's the weather in ny now?"
 document = "Currently New York is rainy."
 # encode the query & document
-feature = tokenizer([query, document], padding=True, truncation=True, return_tensors='pt', return_token_type_ids=False)
 output = model(**feature)[0]
 sparse_vector = get_sparse_vector(feature, output)

 document = "Currently New York is rainy."
 # encode the query & document
+feature = tokenizer([query, document], padding=True, truncation=True, return_tensors='pt')
 output = model(**feature)[0]
 sparse_vector = get_sparse_vector(feature, output)

tokenizer_config.json CHANGED Viewed

@@ -8,6 +8,6 @@
   "sep_token": "[SEP]",
   "strip_accents": null,
   "tokenize_chinese_chars": true,
-  "tokenizer_class": "BertTokenizer",
   "unk_token": "[UNK]"
 }

   "sep_token": "[SEP]",
   "strip_accents": null,
   "tokenize_chinese_chars": true,
+  "tokenizer_class": "DistilBertTokenizer",
   "unk_token": "[UNK]"
 }