Update src/model_loader.py

src/model_loader.py

@@ -9,12 +9,12 @@ cached_tokenizer = None
 def load_model():
     global cached_model, cached_tokenizer
     if cached_model is None or cached_tokenizer is None:
-        # bnb_config = BitsAndBytesConfig(
-            # load_in_4bit=True,
-            # bnb_4bit_use_double_quant=True,
-            # bnb_4bit_quant_type="nf4",
-            # bnb_4bit_compute_dtype=torch.bfloat16
-        # )
+        bnb_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_use_double_quant=True,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.bfloat16
+        )
         cached_model = AutoModelForCausalLM.from_pretrained(MODEL_NAME) #, quantization_config=bnb_config
         cached_tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
     return cached_model, cached_tokenizer