Update app.py

app.py

@@ -1,128 +1,106 @@
 import gradio as gr
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
-import requests
+from transformers import AutoTokenizer, AutoModelForCausalLM
 
-# Verified existing SmilyAI models on Huggingface
-MODEL_CANDIDATES = [
+ # List of available SmilyAI Sam models (adjust as needed)
+MODELS = [
+    "Smilyai-labs/Sam-reason-A1",
     "Smilyai-labs/Sam-reason-S1",
     "Smilyai-labs/Sam-reason-S1.5",
-    "Smilyai-labs/Sam-reason-S2.1",
+    "Smilyai-labs/Sam-reason-S2",
+    "Smilyai-labs/Sam-reason-S3",
     "Smilyai-labs/Sam-reason-v1",
     "Smilyai-labs/Sam-reason-v2",
-    "Smilyai-labs/Sam-large-v1",
-    "Smilyai-labs/Sam-flash-mini-v1",
-    "Smilyai-labs/Sam-reason-A1"
+    "Smilyai-labs/Sam-flash-mini-v1"
 ]
 
-def model_exists(repo_id):
-    url = f"https://huggingface.co/api/models/{repo_id}"
-    try:
-        response = requests.get(url)
-        return response.status_code == 200
-    except Exception:
-        return False
+ device = "cuda" if torch.cuda.is_available() else "cpu"
 
-# Filter models that actually exist
-AVAILABLE_MODELS = [m for m in MODEL_CANDIDATES if model_exists(m)]
+ # Global vars to hold model and tokenizer
+model = None
+tokenizer = None
 
-if not AVAILABLE_MODELS:
-    raise RuntimeError("No verified SmilyAI models are available from Huggingface!")
+ def load_model(model_name):
+    global model, tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
+    model.eval()
+    return f"Loaded model: {model_name}"
 
-device = "cuda" if torch.cuda.is_available() else "cpu"
+ def generate_stream(prompt, max_length=100, temperature=0.7, top_p=0.9):
+    global model, tokenizer
+    if model is None or tokenizer is None:
+        yield "Model not loaded. Please select a model first."
+        return
 
-# Globals to hold current model and tokenizer
-model = None
-tokenizer = None
-generator = None
-
-def load_model(model_name):
-    global model, tokenizer, generator
-    try:
-        tokenizer = AutoTokenizer.from_pretrained(model_name)
-        model = AutoModelForCausalLM.from_pretrained(model_name).to(device)
-        model.eval()
-        # Use pipeline for generation with streaming support
-        generator = pipeline("text-generation", model=model, tokenizer=tokenizer, device=0 if device=="cuda" else -1)
-        return f"✅ Loaded model: {model_name} on {device}"
-    except Exception as e:
-        return f"❌ Failed to load model: {model_name}\n{str(e)}"
-
-def chat_stream(user_message, history, model_name):
-    global model, tokenizer, generator
-
-    if model is None or tokenizer is None or generator is None:
-        load_status = load_model(model_name)
-        if load_status.startswith("❌"):
-            yield history, load_status
-            return
+     input_ids = tokenizer(prompt, return_tensors="pt").input_ids.to(device)
 
-    if history is None:
-        history = []
+     generated_ids = input_ids
+    output_text = tokenizer.decode(input_ids[0])
 
-    # Append user input to history
-    history.append((user_message, ""))
+     # Generate tokens one by one
+    for _ in range(max_length):
+        outputs = model(generated_ids)
+        logits = outputs.logits
 
-    # Prepare prompt with conversation history for multi-turn chat
-    prompt = ""
-    for user, bot in history[:-1]:
-        prompt += f"User: {user}\nSam: {bot}\n"
-    prompt += f"User: {user_message}\nSam:"
+         # Get logits for last token
+        next_token_logits = logits[:, -1, :] / temperature
 
-    # Streaming token generation
-    response_text = ""
-    try:
-        # Set parameters to generate text token by token
-        # Use generator with `stream=True` if supported (Huggingface pipeline streaming)
-        # Note: some transformers versions or models may not support streaming in pipeline.
-        # We'll simulate streaming here by chunking output.
+         # Apply top_p filtering for nucleus sampling
+        sorted_logits, sorted_indices = torch.sort(next_token_logits, descending=True)
+        cumulative_probs = torch.softmax(sorted_logits, dim=-1).cumsum(dim=-1)
 
-        # Generate full text first (fallback)
-        output = generator(prompt, max_new_tokens=128, do_sample=True, top_p=0.9, temperature=0.8)[0]['generated_text']
+         # Remove tokens with cumulative prob above top_p
+        sorted_indices_to_remove = cumulative_probs > top_p
+        # Shift mask right to keep at least one token
+        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
+        sorted_indices_to_remove[..., 0] = 0
 
-        # Extract the new bot output (everything after prompt)
-        bot_reply = output[len(prompt):].strip()
+         filtered_logits = next_token_logits.clone()
+        filtered_logits[:, sorted_indices[sorted_indices_to_remove]] = -float('Inf')
 
-        # Stream output token-by-token to Gradio
-        for i in range(1, len(bot_reply)+1):
-            partial = bot_reply[:i]
-            history[-1] = (user_message, partial)
-            yield history, ""
-    except Exception as e:
-        history[-1] = (user_message, f"Error during generation: {str(e)}")
-        yield history, ""
+         # Sample from filtered distribution
+        probabilities = torch.softmax(filtered_logits, dim=-1)
+        next_token = torch.multinomial(probabilities, num_samples=1)
 
-def reset_chat():
-    return [], ""
+         generated_ids = torch.cat([generated_ids, next_token], dim=-1)
 
-with gr.Blocks() as demo:
-    gr.Markdown("# SmilyAI Sam Chatbot")
+         new_token_text = tokenizer.decode(next_token[0])
+        output_text += new_token_text
 
-    with gr.Row():
-        model_dropdown = gr.Dropdown(choices=AVAILABLE_MODELS, value=AVAILABLE_MODELS[0], label="Select Sam Model")
+         yield output_text
 
-    chatbot = gr.Chatbot()
-    state = gr.State([])  # conversation history
-    status = gr.Textbox(value="", interactive=False, visible=True, label="Status")
+         # Stop if EOS token generated
+        if next_token.item() == tokenizer.eos_token_id:
+            break
 
-    with gr.Row():
-        user_input = gr.Textbox(show_label=False, placeholder="Type your message and hit Enter", lines=2)
-        submit_btn = gr.Button("Send")
+ def on_model_change(model_name):
+    status = load_model(model_name)
+    return status
 
-    def on_submit(user_message, history, model_name):
-        return chat_stream(user_message, history, model_name)
+ with gr.Blocks() as demo:
+    gr.Markdown("# SmilyAI Sam Models — Manual Token Streaming Generator")
 
-    submit_btn.click(on_submit, inputs=[user_input, state, model_dropdown], outputs=[chatbot, status], queue=True)
-    user_input.submit(on_submit, inputs=[user_input, state, model_dropdown], outputs=[chatbot, status], queue=True)
+     with gr.Row():
+        model_selector = gr.Dropdown(choices=MODELS, value=MODELS[0], label="Select Model")
+        status = gr.Textbox(label="Status", interactive=False)
 
-    def on_model_change(new_model):
-        # Reload model on change, reset chat
-        status_message = load_model(new_model)
-        return [], status_message
+     prompt_input = gr.Textbox(lines=3, placeholder="Enter your prompt here...", label="Prompt")
+    output_box = gr.Textbox(label="Generated Text", lines=15, interactive=False)
 
-    model_dropdown.change(on_model_change, inputs=model_dropdown, outputs=[chatbot, status])
+     generate_btn = gr.Button("Generate")
+
+     # Load default model
+    status.value = load_model(MODELS[0])
+
+     model_selector.change(on_model_change, inputs=model_selector, outputs=status)
+
+     def generate_func(prompt):
+        if not prompt.strip():
+            yield "Please enter a prompt."
+            return
+        yield from generate_stream(prompt)
 
-    reset_btn = gr.Button("Reset Chat")
-    reset_btn.click(reset_chat, outputs=[chatbot, status])
+     generate_btn.click(generate_func, inputs=prompt_input, outputs=output_box)
 
-demo.launch()
+ demo.launch()