customstreamer

app.py

@@ -154,8 +154,12 @@ class BSIChatbot:
         Question: {query}"""
 
         input_ids = self.llmtokenizer(final_prompt, return_tensors="pt").to(self.llmpipeline.model.device)
-        for token in self.llmpipeline.model.generate(input_ids, streamer=self.streamer, max_new_tokens=500):
-            yield token
+        streamer=self.streamer
+
+        self.llmpipeline.model.generate(input_ids=input_ids, streamer=streamer, max_new_tokens=500, temperature=0.7, repetition_penalty=1.1)
+
+        for text in streamer:
+            yield text
 
         #input_ids = self.llmtokenizer(final_prompt, return_tensors="pt").to(self.llmpipeline.model.device)
         #streamer = TextIteratorStreamer(self.llmtokenizer, skip_prompt=True)