import gradio as gr from openai import OpenAI import os # Retrieve the access token from the environment variable ACCESS_TOKEN = os.getenv("HF_TOKEN") print("Access token loaded.") # Initialize the OpenAI client with the Hugging Face Inference API endpoint client = OpenAI( base_url="https://api-inference.huggingface.co/v1/", api_key=ACCESS_TOKEN, ) print("OpenAI client initialized.") def respond( message, history: list[tuple[str, str]], system_message, max_tokens, temperature, top_p, frequency_penalty, seed, custom_model ): """ Respond function for ChatInterface. """ print(f"Received message: {message}") print(f"History: {history}") print(f"System message: {system_message}") print(f"Max tokens: {max_tokens}, Temperature: {temperature}, Top-P: {top_p}") print(f"Frequency Penalty: {frequency_penalty}, Seed: {seed}") print(f"Selected model (custom_model): {custom_model}") if seed == -1: seed = None # Construct the messages array messages = [{"role": "system", "content": system_message}] for val in history: user_part = val[0] assistant_part = val[1] if user_part: messages.append({"role": "user", "content": user_part}) if assistant_part: messages.append({"role": "assistant", "content": assistant_part}) messages.append({"role": "user", "content": message}) # If user provided a model, use it; else use default model_to_use = custom_model.strip() if custom_model.strip() != "" else "meta-llama/Llama-3.3-70B-Instruct" response = "" for message_chunk in client.chat.completions.create( model=model_to_use, max_tokens=max_tokens, stream=True, temperature=temperature, top_p=top_p, frequency_penalty=frequency_penalty, seed=seed, messages=messages, ): token_text = message_chunk.choices[0].delta.content response += token_text yield response # ------------------------- # GRADIO UI CONFIGURATION # ------------------------- # Create a Chatbot component chatbot = gr.Chatbot( height=600, show_copy_button=True, placeholder="Select a model and begin chatting", likeable=True, layout="panel" ) # Create textboxes/sliders for system prompt, tokens, etc. system_message_box = gr.Textbox(value="", label="System message") max_tokens_slider = gr.Slider(1, 4096, value=512, step=1, label="Max new tokens") temperature_slider = gr.Slider(0.1, 4.0, value=0.7, step=0.1, label="Temperature") top_p_slider = gr.Slider(0.1, 1.0, value=0.95, step=0.05, label="Top-P") frequency_penalty_slider = gr.Slider(-2.0, 2.0, value=0.0, step=0.1, label="Frequency Penalty") seed_slider = gr.Slider(-1, 65535, value=-1, step=1, label="Seed (-1 for random)") custom_model_box = gr.Textbox(value="", label="Custom Model", info="(Optional) Provide a custom Hugging Face model path. Overrides any selected featured model.") def set_custom_model_from_radio(selected): """ Update the Custom Model textbox when a featured model is selected. """ print(f"Featured model selected: {selected}") return selected # Create a user textbox that we can reference # This will become our "Message" input inside the ChatInterface user_textbox = gr.MultimodalTextbox() # No 'examples' here—because we want to keep the user's parameters unchanged demo = gr.ChatInterface( fn=respond, additional_inputs=[ system_message_box, max_tokens_slider, temperature_slider, top_p_slider, frequency_penalty_slider, seed_slider, custom_model_box ], fill_height=True, chatbot=chatbot, textbox=user_textbox, multimodal=True, concurrency_limit=20, theme="Nymbo/Nymbo_Theme", # No examples parameter used cache_examples=False ) print("ChatInterface object created.") with demo: # Featured models accordion with gr.Accordion("Featured Models", open=False): model_search_box = gr.Textbox( label="Filter Models", placeholder="Search for a featured model...", lines=1 ) models_list = [ "meta-llama/Llama-3.3-70B-Instruct", "meta-llama/Llama-3.2-3B-Instruct", "meta-llama/Llama-3.2-1B-Instruct", "meta-llama/Llama-3.1-8B-Instruct", "NousResearch/Hermes-3-Llama-3.1-8B", "google/gemma-2-27b-it", "google/gemma-2-9b-it", "google/gemma-2-2b-it", "mistralai/Mistral-Nemo-Instruct-2407", "mistralai/Mixtral-8x7B-Instruct-v0.1", "mistralai/Mistral-7B-Instruct-v0.3", "Qwen/Qwen2.5-72B-Instruct", "Qwen/QwQ-32B-Preview", "PowerInfer/SmallThinker-3B-Preview", "HuggingFaceTB/SmolLM2-1.7B-Instruct", "TinyLlama/TinyLlama-1.1B-Chat-v1.0", "microsoft/Phi-3.5-mini-instruct", ] featured_model_radio = gr.Radio( label="Select a model below", choices=models_list, value="meta-llama/Llama-3.3-70B-Instruct", interactive=True ) def filter_models(search_term): filtered = [m for m in models_list if search_term.lower() in m.lower()] return gr.update(choices=filtered) model_search_box.change( fn=filter_models, inputs=model_search_box, outputs=featured_model_radio ) featured_model_radio.change( fn=set_custom_model_from_radio, inputs=featured_model_radio, outputs=custom_model_box ) # Example Prompts accordion with gr.Accordion("Example Prompts", open=False): ex1_btn = gr.Button("Example 1: 'Howdy, partner!'") ex2_btn = gr.Button("Example 2: 'What's your model name and who trained you?'") ex3_btn = gr.Button("Example 3: 'How many R's in Strawberry?'") # Helper function that returns an update for user_textbox def load_example(example_text): return gr.update(value=example_text) ex1_btn.click(fn=lambda: load_example("Howdy, partner!"), inputs=[], outputs=user_textbox) ex2_btn.click(fn=lambda: load_example("What's your model name and who trained you?"), inputs=[], outputs=user_textbox) ex3_btn.click(fn=lambda: load_example("How many R's are there in the word Strawberry?"), inputs=[], outputs=user_textbox) print("Gradio interface initialized.") if __name__ == "__main__": print("Launching the demo application.") demo.launch()