app.py

app.py

@@ -85,9 +85,9 @@ class BSIChatbot:
         return self.vectorstore.similarity_search(query=query, k=20)
 
     def initialize_llm(self):
-        #bnb_config = BitsAndBytesConfig(load_in_8bit=True)
-        #llm = AutoModelForCausalLM.from_pretrained(self.llm_path, quantization_config=bnb_config)
-        llm = AutoModelForCausalLM.from_pretrained(self.llm_path)
+        bnb_config = BitsAndBytesConfig(load_in_8bit=True)
+        llm = AutoModelForCausalLM.from_pretrained(self.llm_path, quantization_config=bnb_config)
+        #llm = AutoModelForCausalLM.from_pretrained(self.llm_path)
         self.llmtokenizer = AutoTokenizer.from_pretrained(self.llm_path)
         self.streamer = TextIteratorStreamer(self.llmtokenizer, skip_prompt=True)
         self.llmpipeline = pipeline(