update quant 8int config

config.py

@@ -22,3 +22,4 @@ SENTENCE_EMBEDDING_FILE = os.path.join(
 # SENTENCE_EMBEDDING_FILE = None
 MODEL_TYPE = "openvino"
 DEVICE_TYPE = "cpu"
+QINT8 = False

prepare.py

@@ -1,5 +1,38 @@
-from sentence_transformers import SentenceTransformer
-from config import MODEL_NAME
+from sentence_transformers import (
+    SentenceTransformer,
+    export_static_quantized_openvino_model,
+    export_dynamic_quantized_onnx_model,
+)
 
-model = SentenceTransformer(MODEL_NAME, backend="openvino")
-model.push_to_hub(MODEL_NAME, create_pr=True)
+MODEL_NAME = "Detomo/cl-nagoya-sup-simcse-ja-nss-v_1_0_3"
+
+
+def export_model(backend="onnx", use_qint8=False):
+    if backend == "openvino":
+        model = SentenceTransformer(MODEL_NAME, backend="openvino")
+        if use_qint8:
+            export_static_quantized_openvino_model(
+                model,
+                quantization_config=None,
+                model_name_or_path=MODEL_NAME,
+                push_to_hub=True,
+            )
+        else:
+            model.push_to_hub(MODEL_NAME)
+    elif backend == "onnx":
+        model = SentenceTransformer(MODEL_NAME, backend="onnx")
+        if use_qint8:
+            export_dynamic_quantized_onnx_model(
+                model, "avx512_vnni", MODEL_NAME, push_to_hub=True
+            )
+        else:
+            model.push_to_hub(MODEL_NAME)
+    else:
+        raise ValueError(f"Invalid backend: {backend}")
+
+
+# Export all combinations
+for backend in ["openvino", "onnx"]:
+    for use_qint8 in [True, False]:
+        print(f"Exporting {backend} model with QINT8={use_qint8}")
+        export_model(backend=backend, use_qint8=use_qint8)

services/sentence_transformer_service.py

@@ -2,7 +2,7 @@ import pickle
 from config import (
     MODEL_NAME, MODEL_TYPE, DEVICE_TYPE,
     SENTENCE_EMBEDDING_FILE,  
-    STANDARD_NAME_MAP_DATA_FILE, SUBJECT_DATA_FILE, DATA_DIR
+    STANDARD_NAME_MAP_DATA_FILE, SUBJECT_DATA_FILE, DATA_DIR, QINT8
 )
 from sentence_transformer_lib.sentence_transformer_helper import SentenceTransformerHelper
 from data_lib.subject_data import SubjectData
@@ -25,9 +25,12 @@ class SentenceTransformerService:
 
         print("Loading models and data...")
         # Load sentence transformer model
-        self.sentenceTransformerHelper = SentenceTransformerHelper(model_name=MODEL_NAME, model_type=MODEL_TYPE)
-        print(f"Loading model {MODEL_NAME} with type {MODEL_TYPE}") 
-
+        print(f"Loading model {MODEL_NAME} with type {MODEL_TYPE} and qint8={QINT8}")
+        self.sentenceTransformerHelper = SentenceTransformerHelper(
+            model_name=MODEL_NAME, 
+            model_type=MODEL_TYPE,
+            qint8=QINT8
+        )
         # Load standard subject dictionary
         self.dic_standard_subject = SubjectData.create_standard_subject_dic_from_file(SUBJECT_DATA_FILE)