Update app.py

app.py

@@ -1,179 +1,391 @@
 import gradio as gr
-import spaces
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
-import torch
-from threading import Thread
+import os
+import requests
+import json
+import time
+from dotenv import load_dotenv
 
-phi4_model_path = "microsoft/Phi-4-reasoning-plus"
+# Load .env file (if it exists)
+load_dotenv()
 
-device = "cuda:0" if torch.cuda.is_available() else "cpu"
-
-phi4_model = AutoModelForCausalLM.from_pretrained(phi4_model_path, device_map="auto", torch_dtype="auto")
-phi4_tokenizer = AutoTokenizer.from_pretrained(phi4_model_path)
-
-@spaces.GPU(duration=60)
-def generate_response(user_message, max_tokens, temperature, top_k, top_p, repetition_penalty, history_state):
-    if not user_message.strip():
-        return history_state, history_state
+def create_deepseek_interface():
+    # Get API keys from environment variables
+    api_key = os.getenv("FW_API_KEY")
+    serphouse_api_key = os.getenv("SERPHOUSE_API_KEY")
+    
+    if not api_key:
+        print("Warning: FW_API_KEY environment variable is not set.")
+    if not serphouse_api_key:
+        print("Warning: SERPHOUSE_API_KEY environment variable is not set.")
+    
+    # Keyword extraction function (LLM-based)
+    def extract_keywords_with_llm(query):
+        if not api_key:
+            return "FW_API_KEY not set for LLM keyword extraction.", query
         
-    # Phi-4 model settings
-    model = phi4_model
-    tokenizer = phi4_tokenizer
-    start_tag = "<|im_start|>"
-    sep_tag = "<|im_sep|>"
-    end_tag = "<|im_end|>"
-
-    # Recommended prompt settings by Microsoft
-    system_message = "Your role as an assistant involves thoroughly exploring questions through a systematic thinking process before providing the final precise and accurate solutions. This requires engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, backtracing, and iteration to develop well-considered thinking process. Please structure your response into two main sections: Thought and Solution using the specified format: <think> {Thought section} </think> {Solution section}. In the Thought section, detail your reasoning process in steps. Each step should include detailed considerations such as analysing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, and revisiting previous steps. In the Solution section, based on various attempts, explorations, and reflections from the Thought section, systematically present the final solution that you deem correct. The Solution section should be logical, accurate, and concise and detail necessary steps needed to reach the conclusion. Now, try to solve the following question through the above guidelines:"
-    prompt = f"{start_tag}system{sep_tag}{system_message}{end_tag}"
-    for message in history_state:
-        if message["role"] == "user":
-            prompt += f"{start_tag}user{sep_tag}{message['content']}{end_tag}"
-        elif message["role"] == "assistant" and message["content"]:
-            prompt += f"{start_tag}assistant{sep_tag}{message['content']}{end_tag}"
-    prompt += f"{start_tag}user{sep_tag}{user_message}{end_tag}{start_tag}assistant{sep_tag}"
-
-    inputs = tokenizer(prompt, return_tensors="pt").to(device)
-
-    do_sample = not (temperature == 1.0 and top_k >= 100 and top_p == 1.0)
-
-    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True)
-
-    # sampling techniques
-    generation_kwargs = {
-        "input_ids": inputs["input_ids"],
-        "attention_mask": inputs["attention_mask"],
-        "max_new_tokens": int(max_tokens),
-        "do_sample": True,
-        "temperature": 0.8,
-        "top_k": int(top_k),
-        "top_p": 0.95,
-        "repetition_penalty": repetition_penalty,
-        "streamer": streamer,
-    }
-
-    thread = Thread(target=model.generate, kwargs=generation_kwargs)
-    thread.start()
-
-    # Stream the response
-    assistant_response = ""
-    new_history = history_state + [
-        {"role": "user", "content": user_message},
-        {"role": "assistant", "content": ""}
-    ]
-    for new_token in streamer:
-        cleaned_token = new_token.replace("<|im_start|>", "").replace("<|im_sep|>", "").replace("<|im_end|>", "")
-        assistant_response += cleaned_token
-        new_history[-1]["content"] = assistant_response.strip()
-        yield new_history, new_history
-
-    yield new_history, new_history
+        # Extract keywords using LLM (DeepSeek model)
+        url = "https://api.fireworks.ai/inference/v1/chat/completions"
+        payload = {
+            "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
+            "max_tokens": 200,
+            "temperature": 0.1,  # Low temperature for consistent results
+            "messages": [
+                {
+                    "role": "system",
+                    "content": "You are a deep thinking AI, you may use extremely long chains of thought to deeply consider the problem and deliberate with yourself via systematic reasoning processes to help come to a correct solution prior to answering. You should enclose your thoughts and internal monologue inside tags, and then provide your solution or response to the problem. Extract key search terms from the user's question that would be effective for web searches. Provide these as a search query with words separated by spaces only, without commas. For example: 'Prime Minister Han Duck-soo impeachment results'"
+                },
+                {
+                    "role": "user",
+                    "content": query
+                }
+            ]
+        }
+        headers = {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}"
+        }
+        
+        try:
+            response = requests.post(url, headers=headers, json=payload)
+            response.raise_for_status()
+            result = response.json()
+            
+            # Extract keywords from response
+            keywords = result["choices"][0]["message"]["content"].strip()
+            
+            # Use original query if keywords are too long or improperly formatted
+            if len(keywords) > 100:
+                return f"Extracted keywords: {keywords}", query
+                
+            return f"Extracted keywords: {keywords}", keywords
+            
+        except Exception as e:
+            print(f"Error during keyword extraction: {str(e)}")
+            return f"Error during keyword extraction: {str(e)}", query
+    
+    # Search function using SerpHouse API
+    def search_with_serphouse(query):
+        if not serphouse_api_key:
+            return "SERPHOUSE_API_KEY is not set."
+        
+        try:
+            # Extract keywords
+            extraction_result, search_query = extract_keywords_with_llm(query)
+            print(f"Original query: {query}")
+            print(extraction_result)
+            
+            # Basic GET method seems best after analyzing documentation
+            url = "https://api.serphouse.com/serp/live"
+            
+            # Check if query is in Korean
+            is_korean = any('\uAC00' <= c <= '\uD7A3' for c in search_query)
+            
+            # Simplified parameters
+            params = {
+                "q": search_query,
+                "domain": "google.com",
+                "serp_type": "web",  # Changed to basic web search
+                "device": "desktop",
+                "lang": "ko" if is_korean else "en"
+            }
+            
+            headers = {
+                "Authorization": f"Bearer {serphouse_api_key}"
+            }
+            
+            print(f"Calling SerpHouse API with basic GET method...")
+            print(f"Search term: {search_query}")
+            print(f"Request URL: {url} - Parameters: {params}")
+            
+            # Execute GET request
+            response = requests.get(url, headers=headers, params=params)
+            response.raise_for_status()
+            
+            print(f"SerpHouse API response status code: {response.status_code}")
+            search_results = response.json()
+            
+            # Check response structure
+            print(f"Response structure: {list(search_results.keys()) if isinstance(search_results, dict) else 'Not a dictionary'}")
+            
+            # Parse and format search results (in Markdown)
+            formatted_results = []
+            formatted_results.append(f"## Search term: {search_query}\n\n")
+            
+            # Handle various possible response structures
+            organic_results = None
+            
+            # Possible response structure 1
+            if "results" in search_results and "organic" in search_results["results"]:
+                organic_results = search_results["results"]["organic"]
+                
+            # Possible response structure 2
+            elif "organic" in search_results:
+                organic_results = search_results["organic"]
+                
+            # Possible response structure 3 (nested results)
+            elif "results" in search_results and "results" in search_results["results"]:
+                if "organic" in search_results["results"]["results"]:
+                    organic_results = search_results["results"]["results"]["organic"]
+            
+            # Process organic results if available
+            if organic_results and len(organic_results) > 0:
+                # Output response structure
+                print(f"First organic result structure: {organic_results[0].keys() if len(organic_results) > 0 else 'empty'}")
+                
+                for i, result in enumerate(organic_results[:5], 1):  # Show only top 5 results
+                    title = result.get("title", "No title")
+                    snippet = result.get("snippet", "No content")
+                    link = result.get("link", "#")
+                    displayed_link = result.get("displayed_link", link)
+                    
+                    # Format in Markdown (including number and link)
+                    formatted_results.append(
+                        f"### {i}. [{title}]({link})\n\n"
+                        f"{snippet}\n\n"
+                        f"**Source**: [{displayed_link}]({link})\n\n"
+                        f"---\n\n"
+                    )
+                
+                print(f"Found {len(organic_results)} search results")
+                return "".join(formatted_results)
+            
+            # Handle case with no results or unexpected structure
+            print("No search results or unexpected response structure")
+            print(f"Detailed response structure: {search_results.keys() if hasattr(search_results, 'keys') else 'Unclear structure'}")
+            
+            # Find error messages in response
+            error_msg = "No search results found or response format is different than expected"
+            if "error" in search_results:
+                error_msg = search_results["error"]
+            elif "message" in search_results:
+                error_msg = search_results["message"]
+                
+            return f"## Results for '{search_query}'\n\n{error_msg}"
+            
+        except Exception as e:
+            error_msg = f"Error during search: {str(e)}"
+            print(error_msg)
+            import traceback
+            print(traceback.format_exc())
+            
+            # Add API request details for debugging (in Markdown)
+            return f"## Error Occurred\n\n" + \
+                   f"An error occurred during search: **{str(e)}**\n\n" + \
+                   f"### API Request Details:\n" + \
+                   f"- **URL**: {url}\n" + \
+                   f"- **Search Term**: {search_query}\n" + \
+                   f"- **Parameters**: {params}\n"
+    
+    # Function to call DeepSeek API with streaming
+    def query_deepseek_streaming(message, history, use_deep_research):
+        if not api_key:
+            yield history, "Environment variable FW_API_KEY is not set. Please check the environment variables on the server."
+            return
+        
+        search_context = ""
+        search_info = ""
+        if use_deep_research:
+            try:
+                # Start search (first message)
+                yield history + [(message, "🔍 Extracting optimal keywords and searching the web...")], ""
+                
+                # Execute search - add logs for debugging
+                print(f"Deep Research activated: Starting search for '{message}'")
+                search_results = search_with_serphouse(message)
+                print(f"Search results received: {search_results[:100]}...")  # Output first part of results
+                
+                if not search_results.startswith("Error during search") and not search_results.startswith("SERPHOUSE_API_KEY"):
+                    search_context = f"""
+Here are recent search results related to the user's question. Use this information to provide an accurate response with the latest information:
 
-example_messages = {
-    "Math reasoning": "If a rectangular prism has a length of 6 cm, a width of 4 cm, and a height of 5 cm, what is the length of the longest line segment that can be drawn from one vertex to another?",
-    "Logic puzzle": "Four people (Alex, Blake, Casey, and Dana) each have a different favorite color (red, blue, green, yellow) and a different favorite fruit (apple, banana, cherry, date). Given the following clues: 1) The person who likes red doesn't like dates. 2) Alex likes yellow. 3) The person who likes blue likes cherries. 4) Blake doesn't like apples or bananas. 5) Casey doesn't like yellow or green. Who likes what color and what fruit?",
-    "Physics problem": "A ball is thrown upward with an initial velocity of 15 m/s from a height of 2 meters above the ground. Assuming the acceleration due to gravity is 9.8 m/s², determine: 1) The maximum height the ball reaches. 2) The total time the ball is in the air before hitting the ground. 3) The velocity with which the ball hits the ground."
-}
+{search_results}
 
-with gr.Blocks(theme=gr.themes.Soft()) as demo:
-    gr.Markdown(
-        """
-        # Phi-4-reasoning-plus Chatbot 
-        Welcome to the Phi-4-reasoning-plus Chatbot! This model excels at multi-step reasoning tasks in mathematics, logic, and science.
+Based on the above search results, answer the user's question. If you cannot find a clear answer in the search results, use your knowledge to provide the best answer.
+When citing search results, mention the source, and ensure your answer reflects the latest information.
+"""
+                    search_info = f"🔍 Deep Research feature activated: Generating response based on relevant web search results..."
+                else:
+                    print(f"Search failed or no results: {search_results}")
+            except Exception as e:
+                print(f"Exception occurred during Deep Research: {str(e)}")
+                search_info = f"🔍 Deep Research feature error: {str(e)}"
+        
+        # Prepare conversation history for API request
+        messages = []
+        for user, assistant in history:
+            messages.append({"role": "user", "content": user})
+            messages.append({"role": "assistant", "content": assistant})
+        
+        # Add system message with search context if available
+        if search_context:
+            # DeepSeek model supports system messages
+            messages.insert(0, {"role": "system", "content": search_context})
         
-        The model will provide responses with two sections:
-        1. **<think>**: A detailed step-by-step reasoning process showing its work
-        2. **Solution**: A concise, accurate final answer based on the reasoning
+        # Add new user message
+        messages.append({"role": "user", "content": message})
         
-        Try the example problems below to see how the model breaks down complex reasoning problems.
-        """
-    )
+        # Prepare API request
+        url = "https://api.fireworks.ai/inference/v1/chat/completions"
+        payload = {
+            "model": "accounts/fireworks/models/llama4-maverick-instruct-basic",
+            "max_tokens": 20480,
+            "top_p": 1,
+            "top_k": 40,
+            "presence_penalty": 0,
+            "frequency_penalty": 0,
+            "temperature": 0.6,
+            "messages": messages,
+            "stream": True  # Enable streaming
+        }
+        headers = {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {api_key}"
+        }
+        
+        try:
+            # Request streaming response
+            response = requests.request("POST", url, headers=headers, data=json.dumps(payload), stream=True)
+            response.raise_for_status()  # Raise exception for HTTP errors
+            
+            # Add message and start with initial response
+            new_history = history.copy()
+            
+            # Include search_info in starting message if available
+            start_msg = search_info if search_info else ""
+            new_history.append((message, start_msg))
+            
+            # Full response text
+            full_response = start_msg
+            
+            # Process streaming response
+            for line in response.iter_lines():
+                if line:
+                    line_text = line.decode('utf-8')
+                    
+                    # Remove 'data: ' prefix
+                    if line_text.startswith("data: "):
+                        line_text = line_text[6:]
+                    
+                    # Check for stream end message
+                    if line_text == "[DONE]":
+                        break
+                    
+                    try:
+                        # Parse JSON
+                        chunk = json.loads(line_text)
+                        chunk_content = chunk.get("choices", [{}])[0].get("delta", {}).get("content", "")
+                        
+                        if chunk_content:
+                            full_response += chunk_content
+                            # Update chat history
+                            new_history[-1] = (message, full_response)
+                            yield new_history, ""
+                    except json.JSONDecodeError:
+                        continue
+            
+            # Return final response
+            yield new_history, ""
+            
+        except requests.exceptions.RequestException as e:
+            error_msg = f"API error: {str(e)}"
+            if hasattr(e, 'response') and e.response and e.response.status_code == 401:
+                error_msg = "Authentication failed. Please check your FW_API_KEY environment variable."
+            yield history, error_msg
     
-    history_state = gr.State([])
-
-    with gr.Row():
-        with gr.Column(scale=1):
-            gr.Markdown("### Settings")
-            max_tokens_slider = gr.Slider(
-                minimum=64,
-                maximum=32768,
-                step=1024,
-                value=4096,
-                label="Max Tokens"
-            )
-            with gr.Accordion("Advanced Settings", open=False):
-                temperature_slider = gr.Slider(
-                    minimum=0.1,
-                    maximum=2.0,
-                    value=0.8,
-                    label="Temperature"
-                )
-                top_k_slider = gr.Slider(
-                    minimum=1,
-                    maximum=100,
-                    step=1,
-                    value=50,
-                    label="Top-k"
-                )
-                top_p_slider = gr.Slider(
-                    minimum=0.1,
-                    maximum=1.0,
-                    value=0.95,
-                    label="Top-p"
+    # Create Gradio interface
+    with gr.Blocks(theme="soft", fill_height=True) as demo:
+        # Header section
+        gr.Markdown(
+            """
+            # 🤖 Llama-4-Maverick-17B + Research 
+            ### Llama-4-Maverick-17B Model + Real-time 'Deep Research' Agentic AI System @ https://discord.gg/openfreeai
+            """
+        )
+        
+        # Main layout
+        with gr.Row():
+            # Main content area
+            with gr.Column():
+                # Chat interface
+                chatbot = gr.Chatbot(
+                    height=500,
+                    show_label=False,
+                    container=True
                 )
-                repetition_penalty_slider = gr.Slider(
-                    minimum=1.0,
-                    maximum=2.0,
-                    value=1.0,
-                    label="Repetition Penalty"
+                
+                # Add Deep Research toggle and status display
+                with gr.Row():
+                    with gr.Column(scale=3):
+                        use_deep_research = gr.Checkbox(
+                            label="Enable Deep Research",
+                            info="Utilize optimal keyword extraction and web search for latest information",
+                            value=False
+                        )
+                    with gr.Column(scale=1):
+                        api_status = gr.Markdown("API Status: Ready")
+                
+                # Check and display API key status
+                if not serphouse_api_key:
+                    api_status.value = "⚠️ SERPHOUSE_API_KEY is not set"
+                if not api_key:
+                    api_status.value = "⚠️ FW_API_KEY is not set"
+                if api_key and serphouse_api_key:
+                    api_status.value = "✅ API keys configured"
+                
+                # Input area
+                with gr.Row():
+                    msg = gr.Textbox(
+                        label="Message",
+                        placeholder="Enter your prompt here...",
+                        show_label=False,
+                        scale=9
+                    )
+                    submit = gr.Button("Send", variant="primary", scale=1)
+                
+                # Clear conversation button
+                with gr.Row():
+                    clear = gr.ClearButton([msg, chatbot], value="🧹 Clear Conversation")
+                    
+                # Example queries
+                gr.Examples(
+                    examples=[
+                        "Explain the difference between Transformers and RNNs in deep learning.",
+                        "Write a Python function to find prime numbers within a specific range.",
+                        "Summarize the key concepts of reinforcement learning."
+                    ],
+                    inputs=msg
                 )
+                
+                # Error message display
+                error_box = gr.Markdown("")
         
-        with gr.Column(scale=4):
-            chatbot = gr.Chatbot(label="Chat", type="messages")
-            with gr.Row():
-                user_input = gr.Textbox(
-                    label="Your message",
-                    placeholder="Type your message here...",
-                    scale=3
-                )
-                submit_button = gr.Button("Send", variant="primary", scale=1)
-                clear_button = gr.Button("Clear", scale=1)
-            gr.Markdown("**Try these examples:**")
-            with gr.Row():
-                example1_button = gr.Button("Math reasoning")
-                example2_button = gr.Button("Logic puzzle")
-                example3_button = gr.Button("Physics problem")
-
-    submit_button.click(
-        fn=generate_response,
-        inputs=[user_input, max_tokens_slider, temperature_slider, top_k_slider, top_p_slider, repetition_penalty_slider, history_state],
-        outputs=[chatbot, history_state]
-    ).then(
-        fn=lambda: gr.update(value=""),
-        inputs=None,
-        outputs=user_input
-    )
-
-    clear_button.click(
-        fn=lambda: ([], []),
-        inputs=None,
-        outputs=[chatbot, history_state]
-    )
-
-    example1_button.click(
-        fn=lambda: gr.update(value=example_messages["Math reasoning"]),
-        inputs=None,
-        outputs=user_input
-    )
-    example2_button.click(
-        fn=lambda: gr.update(value=example_messages["Logic puzzle"]),
-        inputs=None,
-        outputs=user_input
-    )
-    example3_button.click(
-        fn=lambda: gr.update(value=example_messages["Physics problem"]),
-        inputs=None,
-        outputs=user_input
-    )
+        # Connect buttons to functions
+        submit.click(
+            query_deepseek_streaming,
+            inputs=[msg, chatbot, use_deep_research],
+            outputs=[chatbot, error_box]
+        ).then(
+            lambda: "",
+            None,
+            [msg]
+        )
+        
+        # Allow Enter key submission
+        msg.submit(
+            query_deepseek_streaming,
+            inputs=[msg, chatbot, use_deep_research],
+            outputs=[chatbot, error_box]
+        ).then(
+            lambda: "",
+            None,
+            [msg]
+        )
+        
+    return demo
 
-demo.launch(ssr_mode=False)
+# Run interface
+if __name__ == "__main__":
+    demo = create_deepseek_interface()
+    demo.launch(debug=True)