Use the DistilBertTokenizer for this DistilBERT-based model (#4)

- Use the DistilBertTokenizer for this DistilBERT-based model (e4469d87867f92a54ee361a46db996b0d96a1fad)

README.md

@@ -102,7 +102,7 @@ query = "What's the weather in ny now?"
 document = "Currently New York is rainy."
 
 # encode the query
-feature_query = tokenizer([query], padding=True, truncation=True, return_tensors='pt', return_token_type_ids=False)
+feature_query = tokenizer([query], padding=True, truncation=True, return_tensors='pt')
 input_ids = feature_query["input_ids"]
 batch_size = input_ids.shape[0]
 query_vector = torch.zeros(batch_size, tokenizer.vocab_size)
@@ -110,7 +110,7 @@ query_vector[torch.arange(batch_size).unsqueeze(-1), input_ids] = 1
 query_sparse_vector = query_vector*idf
 
 # encode the document
-feature_document = tokenizer([document], padding=True, truncation=True, return_tensors='pt', return_token_type_ids=False)
+feature_document = tokenizer([document], padding=True, truncation=True, return_tensors='pt')
 output = model(**feature_document)[0]
 document_sparse_vector = get_sparse_vector(feature_document, output)
 

tokenizer_config.json

@@ -8,6 +8,6 @@
   "sep_token": "[SEP]",
   "strip_accents": null,
   "tokenize_chinese_chars": true,
-  "tokenizer_class": "BertTokenizer",
+  "tokenizer_class": "DistilBertTokenizer",
   "unk_token": "[UNK]"
 }