Update app.py

app.py

@@ -7,13 +7,11 @@ import json
 import uuid
 import os
 
-
-
 token=os.environ.get("HF_TOKEN")
 username="omnibus"
 dataset_name="tmp"
 api=HfApi(token="")
-client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
+VERBOSE=False
 
 history = []
 hist_out= []
@@ -22,6 +20,30 @@ main_point=[]
 summary.append("")
 main_point.append("")
 
+models=[
+    "google/gemma-7b",
+    "google/gemma-7b-it",
+    "google/gemma-2b",
+    "google/gemma-2b-it",
+    "meta-llama/Llama-2-7b-chat-hf",
+    "codellama/CodeLlama-70b-Instruct-hf",
+    "openchat/openchat-3.5-0106",
+    "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
+    "mistralai/Mixtral-8x7B-Instruct-v0.1",
+    "mistralai/Mixtral-8x7B-Instruct-v0.2",
+]
+
+client_z=[]
+
+def load_models(inp):
+    if VERBOSE==True:    
+        print(type(inp))
+        print(inp)
+        print(models[inp])
+    client_z.clear()
+    client_z.append(InferenceClient(models[inp]))
+    return gr.update(label=models[inp])
+
 def format_prompt(message, history):
   prompt = "<s>"
   for user_prompt, bot_response in history:
@@ -64,6 +86,7 @@ def compress_history(formatted_prompt):
     #history.append((prompt,""))
     #formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
     formatted_prompt = formatted_prompt
+    client=client_z[0]
     stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
     output = ""
 
@@ -96,7 +119,10 @@ def question_generate(prompt, history, agent_name=agents[0], sys_prompt="", temp
     )
     #history.append((prompt,""))
     formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
+    client=client_z[0]
+
     stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+        
     output = ""
 
     for response in stream:
@@ -126,6 +152,7 @@ def blog_poster_reply(prompt, history, agent_name=agents[0], sys_prompt="", temp
     )
     #history.append((prompt,""))
     formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
+    client=client_z[0]
     stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
     output = ""
 
@@ -180,7 +207,7 @@ def load_html(inp,title):
 
 
 
-def generate(prompt, history, agent_name=agents[0], sys_prompt="", temperature=0.9, max_new_tokens=1048, top_p=0.95, repetition_penalty=1.0,):
+def generate(prompt, history, agent_name=agents[0], sys_prompt="", temperature=0.9, max_new_tokens=1048, top_p=0.95, repetition_penalty=1.0, m_choice):
     html_out=""
     #main_point[0]=prompt
     #print(datetime.datetime.now())
@@ -241,7 +268,7 @@ def generate(prompt, history, agent_name=agents[0], sys_prompt="", temperature=0
         if len(formatted_prompt) < (40000):
             print(len(formatted_prompt))
 
-            
+            client=client_z[0]
             stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
             output = ""
             #if history:
@@ -325,16 +352,19 @@ with gr.Blocks() as app:
         submit_b = gr.Button()
         stop_b = gr.Button("Stop")
         clear = gr.ClearButton([msg, chatbot])
+    m_choice=gr.Dropdown(label="Models",type='index',choices=[c for c in models],value=models[0],interactive=True)
+
     sumbox=gr.Textbox("Summary", max_lines=100)
     with gr.Column():
         sum_out_box=gr.JSON(label="Summaries")
         hist_out_box=gr.JSON(label="History")
-        
-    sub_b = submit_b.click(generate, [msg,chatbot],[msg,chatbot,sumbox,sum_out_box,hist_out_box,html])
-    sub_e = msg.submit(generate, [msg, chatbot], [msg, chatbot,sumbox,sum_out_box,hist_out_box,html])
-    stop_b.click(None,None,None, cancels=[sub_b,sub_e])
-    
+
+
+    client_choice.change(load_models,client_choice,[chat_b])
+    app.load(load_models,client_choice,[chat_b]).then(load_html,None,html)
     
+    sub_b = submit_b.click(generate, [msg,chatbot],[msg,chatbot,sumbox,sum_out_box,hist_out_box,html,m_choice])
+    sub_e = msg.submit(generate, [msg, chatbot], [msg, chatbot,sumbox,sum_out_box,hist_out_box,html,m_choice])
+    stop_b.click(None,None,None, cancels=[sub_b,sub_e])
     
-    app.load(load_html,None,html)
 app.queue(default_concurrency_limit=20).launch()