Update app.py

app.py

@@ -1,10 +1,8 @@
-from unsloth import FastLanguageModel
 import torch
 import gradio as gr
 from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
 from threading import Thread
 
-
 # Load model and tokenizer once at startup
 model_name = "jsbeaudry/makandal-v2"
 tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -14,13 +12,9 @@ model = AutoModelForCausalLM.from_pretrained(
     device_map="auto"
 )
 
-# Prepare model for inference
-FastLanguageModel.for_inference(model)
-
 think_token_id = tokenizer.convert_tokens_to_ids("</think>")
 
 def generate_response_stream(prompt):
-    """Generator function that yields streaming responses"""
     # Format input for chat template
     messages = [{"role": "user", "content": prompt}]
     text = tokenizer.apply_chat_template(
@@ -34,7 +28,7 @@ def generate_response_stream(prompt):
     model_inputs = tokenizer([text], return_tensors="pt")
     model_inputs = {k: v.to(model.device) for k, v in model_inputs.items()}
     
-    # Setup streamer
+    # Create streamer
     text_streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
     
     # Generation parameters
@@ -53,52 +47,22 @@ def generate_response_stream(prompt):
     thread.start()
     
     # Stream the response
-    full_response = ""
-    thinking_content = ""
-    content = ""
-    
+    partial_response = ""
     for new_text in text_streamer:
-        full_response += new_text
-        
-        # Check if we've hit the think token
-        if "</think>" in full_response:
-            parts = full_response.split("</think>", 1)
-            thinking_content = parts[0].strip()
-            content = parts[1].strip() if len(parts) > 1 else ""
-            yield thinking_content, content
-        else:
-            # If no think token yet, everything is thinking content
-            thinking_content = full_response.strip()
-            yield thinking_content, content
-    
-    # Final yield with complete response
-    if "</think>" in full_response:
-        parts = full_response.split("</think>", 1)
-        thinking_content = parts[0].strip()
-        content = parts[1].strip() if len(parts) > 1 else ""
-    else:
-        # If no think token found, treat everything as content
-        thinking_content = ""
-        content = full_response.strip()
+        partial_response += new_text
+        yield partial_response
     
-    yield thinking_content, content
-
-def generate_response_interface(prompt):
-    """Interface function for Gradio that handles streaming"""
-    for thinking, content in generate_response_stream(prompt):
-        yield thinking, content
+    # Wait for thread to complete
+    thread.join()
 
 # Gradio Interface with streaming
 demo = gr.Interface(
-    fn=generate_response_interface,
+    fn=generate_response_stream,
     inputs=gr.Textbox(lines=2, placeholder="Ekri yon sijè oswa yon fraz..."),
-    outputs=[
-        gr.Textbox(label="Thinking Content", interactive=False),
-        gr.Textbox(label="Respons", interactive=False)
-    ],
+    outputs=gr.Textbox(label="Respons"),
     title="Makandal Text Generator (Streaming)",
     description="Ekri yon fraz oswa mo kle pou jenere tèks ak modèl Makandal la. Modèl sa fèt espesyalman pou kontèks Ayiti.",
-    live=False  # Set to True if you want real-time updates as user types
+    live=False  # Set to False to prevent auto-triggering
 )
 
 if __name__ == "__main__":
@@ -107,14 +71,6 @@ if __name__ == "__main__":
 
 
 
-
-
-
-
-
-
-
-
 # import torch
 # import gradio as gr
 # from transformers import AutoTokenizer, AutoModelForCausalLM