disable share button for now

app.py

@@ -13,6 +13,10 @@ API_TOKEN = os.environ.get("API_TOKEN", None)
 STAR_CHAT_API_URL = os.environ.get("STAR_CHAT_API_URL", None)
 STAR_CHAT_GPT_API_URL = os.environ.get("STAR_CHAT_GPT_API_URL", None)
 
+API_TOKEN = "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"
+STAR_CHAT_API_URL = "https://i1qe9e7uv7jzsg8k.us-east-1.aws.endpoints.huggingface.cloud"
+STAR_CHAT_GPT_API_URL = "https://czpdnzuklyfoqjbs.us-east-1.aws.endpoints.huggingface.cloud"
+
 model_to_api = {
     "StarChat": STAR_CHAT_API_URL,
     "StarChatGPT": STAR_CHAT_GPT_API_URL,
@@ -400,6 +404,6 @@ with gr.Blocks(theme=theme, analytics_enabled=False, css=css) as demo:
     clear_chat_button.click(clear_chat, outputs=[chatbot, history])
 
     model.change(radio_on_change, outputs=[chatbot, history])
-    share_button.click(None, [], [], _js=share_js)
+    # share_button.click(None, [], [], _js=share_js)
 
 demo.queue(concurrency_count=16).launch(debug=True)

arena.py

@@ -0,0 +1,491 @@
+import os
+import random
+from itertools import zip_longest
+
+import gradio as gr
+from text_generation import Client, InferenceAPIClient
+
+from share_btn import community_icon_html, loading_icon_html, share_js, share_btn_css
+
+TOKEN = os.environ.get("API_TOKEN", None)
+
+# 13 models
+model_names = [
+    # "vicuna-13b",
+    # "koala-13b",
+    "oasst-pythia-12b",
+    # "alpaca-13b",
+    # "chatglm-6b",
+    # "llama-13b",
+    # "stablelm-tuned-alpha-7b",
+    "bloom",
+    "bloomz",
+    "flan-t5-xxl",
+    "flan-ul2",
+    "santacoder",
+    "gpt-neox-20b"
+]
+
+model_path = [
+    # "HuggingFaceH4/stable-vicuna-13b-2904",
+    # "TheBloke/koala-13B-HF",
+    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
+    # "anon8231489123/gpt4-x-alpaca-13b-native-4bit-128g",
+    # "THUDM/chatglm-6b-int4-qe",
+    # "decapoda-research/llama-13b-hf",
+    # "stabilityai/stablelm-tuned-alpha-7b",
+    "bigscience/bloom",
+    "bigscience/bloomz",
+    "google/flan-t5-xxl",
+    "google/flan-ul2",
+    "bigcode/santacoder",
+    "EleutherAI/gpt-neox-20b"
+]
+
+model_endpoints = [
+    "https://huggingface.co/HuggingFaceH4/stable-vicuna-13b-2904",
+    "https://huggingface.co/TheBloke/koala-13B-HF",
+    "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
+    "https://huggingface.co/anon8231489123/gpt4-x-alpaca-13b-native-4bit-128g",
+    "https://huggingface.co/THUDM/chatglm-6b-int4-qe",
+    "https://huggingface.co/decapoda-research/llama-13b-hf",
+    "https://huggingface.co/stabilityai/stablelm-tuned-alpha-7b",
+    "https://huggingface.co/bigscience/bloom",
+    "https://huggingface.co/bigscience/bloomz",
+    "https://huggingface.co/google/flan-t5-xxl",
+    "https://huggingface.co/google/flan-ul2",
+    "https://huggingface.co/bigcode/santacoder",
+    "https://huggingface.co/EleutherAI/gpt-neox-20b"
+]
+
+model_descriptions = [
+    "An open-source chatbot trained by fine-tuning LLaMA on user-shared conversations collected from ShareGPT by LMSYS",  # vicuna
+    "A 13B model dialogue model created at Berkeley",  # koala
+    "A Pythia 12B fine-tuned on human demonstrations of assistant conversations collected through the https://open-assistant.io/",  # oasst
+    "An instruction-following LLaMA model released by Stanford",  # alpaca
+    "A 6B open bilingual language model based on General Language Model (GLM) framework",  # chatglm
+    "A large language model released by Meta AI",  # llama
+    "A 7B model released by Stability AI",  # stablelm
+    "BigScience Large Open-science Open-access Multilingual Language Model",  # bloom
+    "A family of models capable of following human instructions in dozens of languages zero-shot finetuned from BLOOM & mT5",  # bloomz
+    "An enhanced version of T5 that has been finetuned in a mixture of tasks",  # flan-t5
+    "An encoder decoder model based on the T5 architecture",  # flan-ul2
+    # santacoder
+    "A series of 1.1B parameter models trained on the Python, Java, and JavaScript subset of The Stack (v1.1)",
+    "A 20 billion parameter autoregressive language model trained on the Pile using the GPT-NeoX library"  # gpt-neox-20b
+]
+
+model_to_path_dict = dict([(name, path) for name, path in zip(model_names, model_path)])
+model_to_endpoint_dict = dict([(name, endpoint)
+                              for name, endpoint in zip(model_names, model_endpoints)])
+model_list = sorted(model_names, key=str.casefold)
+table_data = [[name, desc, source]
+              for name, desc, source in zip(model_names, model_descriptions, model_path)]
+
+CHATBOT_A_DEFAULT = "bloom"
+CHATBOT_B_DEFAULT = "bloomz"
+
+theme = gr.themes.Monochrome(
+    primary_hue="indigo",
+    secondary_hue="blue",
+    neutral_hue="slate",
+    radius_size=gr.themes.sizes.radius_sm,
+    font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"],
+)
+
+openchat_preprompt = (
+    "\n<human>: Hi!\n<bot>: My name is Bot, model version is 0.15, part of an open-source kit for "
+    "fine-tuning new bots! I was created by Together, LAION, and Ontocord.ai and the open-source "
+    "community. I am not human, not evil and not alive, and thus have no thoughts and feelings, "
+    "but I am programmed to be helpful, polite, honest, and friendly.\n"
+)
+
+
+def get_client(model: str):
+    if model in ["vicuna-13b", "koala-13b", "oasst-pythia-12b", "alpaca-13b", "chatglm-6b", "llama-13b", "stablelm-tuned-alpha-7b"]:
+        print("JUST CLIENT")
+        return Client(model_to_endpoint_dict[model])
+    return InferenceAPIClient(model_to_path_dict[model], token=TOKEN)
+
+
+def get_usernames(model: str):
+    """
+    Returns:
+        (str, str, str, str): pre-prompt, username, bot name, separator
+    """
+    if model in ("OpenAssistant/oasst-sft-1-pythia-12b", "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"):
+        return "", "<|prompter|>", "<|assistant|>", "<|endoftext|>"
+    if model == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
+        return openchat_preprompt, "<human>: ", "<bot>: ", "\n"
+    return "", "User: ", "Assistant: ", "\n"
+
+
+def get_total_inputs(inputs, chatbot, preprompt, user_name, assistant_name, sep):
+    past = []
+    for data in chatbot:
+        user_data, model_data = data
+
+        if not user_data.startswith(user_name):
+            user_data = user_name + user_data
+        if not model_data.startswith(sep + assistant_name):
+            model_data = sep + assistant_name + model_data
+
+        past.append(user_data + model_data.rstrip() + sep)
+
+    if not inputs.startswith(user_name):
+        inputs = user_name + inputs
+
+    total_inputs = preprompt + "".join(past) + inputs + sep + assistant_name.rstrip()
+
+    return total_inputs
+
+
+def get_iterator(model, client, total_inputs, user_name, assistant_name, temperature, max_new_tokens, top_p, repetition_penalty):
+    if model in ("OpenAssistant/oasst-sft-1-pythia-12b"):
+        iterator = client.generate_stream(
+            total_inputs,
+            typical_p=0.2,  # fixed
+            truncate=1000,
+            max_new_tokens=max_new_tokens,
+        )
+    else:
+        iterator = client.generate_stream(
+            total_inputs,
+            top_p=top_p if top_p < 1.0 else None,
+            top_k=50,
+            truncate=1000,
+            repetition_penalty=repetition_penalty,
+            temperature=temperature,
+            max_new_tokens=max_new_tokens,
+            stop_sequences=[user_name.rstrip(), assistant_name.rstrip()],
+        )
+    return iterator
+
+
+def has_no_history(chatbot_a, chatbot_b, history_a, history_b):
+    return not chatbot_a and not history_a and not chatbot_b and not history_b
+
+
+def generate(inputs,
+             model_a,
+             chatbot_a,
+             history_a,
+             model_b,
+             chatbot_b,
+             history_b,
+             temperature=0.9,
+             max_new_tokens=256,
+             top_p=0.25,
+             repetition_penalty=1.0,
+             do_save=True):
+
+    # Don't return meaningless message when the input is empty
+    if not inputs:
+        yield chatbot_a, chatbot_b, history_a, history_b, inputs, ""
+
+    client_a = get_client(model_a)
+    client_b = get_client(model_b)
+
+    preprompt_a, user_name_a, assistant_name_a, sep_a = get_usernames(model_a)
+    preprompt_b, user_name_b, assistant_name_b, sep_b = get_usernames(model_b)
+
+    history_a.append(inputs)
+    history_b.append(inputs)
+
+    total_inputs_a = get_total_inputs(inputs, chatbot_a, preprompt_a,
+                                      user_name_a, assistant_name_a, sep_a)
+    total_inputs_b = get_total_inputs(inputs, chatbot_b, preprompt_b,
+                                      user_name_b, assistant_name_b, sep_b)
+
+    partial_words_a = ""
+    partial_words_b = ""
+    iterator_a = get_iterator(model_a, client_a, total_inputs_a, user_name_a,
+                              assistant_name_a, temperature, max_new_tokens, top_p, repetition_penalty)
+    iterator_b = get_iterator(model_b, client_b, total_inputs_b, user_name_b,
+                              assistant_name_b, temperature, max_new_tokens, top_p, repetition_penalty)
+    for i, (response_a, response_b) in enumerate(zip_longest(iterator_a, iterator_b, fillvalue="")):
+        text_a = "" if not response_a or response_a.token.special else response_a.token.text
+        text_b = "" if not response_b or response_b.token.special else response_b.token.text
+
+        partial_words_a = partial_words_a + text_a
+        partial_words_b = partial_words_b + text_b
+        if partial_words_a.endswith(user_name_a.rstrip()):
+            partial_words_a = partial_words_a.rstrip(user_name_a.rstrip())
+        if partial_words_b.endswith(user_name_b.rstrip()):
+            partial_words_b = partial_words_b.rstrip(user_name_b.rstrip())
+
+        if partial_words_a.endswith(assistant_name_a.rstrip()):
+            partial_words_a = partial_words_a.rstrip(assistant_name_a.rstrip())
+        if partial_words_b.endswith(assistant_name_b.rstrip()):
+            partial_words_b = partial_words_b.rstrip(assistant_name_b.rstrip())
+
+        if i == 0:
+            history_a.append(" " + partial_words_a)
+            history_b.append(" " + partial_words_b)
+
+        if text_a not in user_name_a:
+            history_a[-1] = partial_words_a
+
+        if text_b not in user_name_b:
+            history_b[-1] = partial_words_b
+
+        chat_a = [
+            (history_a[i].strip(), history_a[i + 1].strip())
+            for i in range(0, len(history_a) - 1, 2)
+        ]
+
+        chat_b = [
+            (history_b[i].strip(), history_b[i + 1].strip())
+            for i in range(0, len(history_b) - 1, 2)
+        ]
+
+        # return inputs to store the latest input in last_user_message and an empty string to clear out message input textbox
+        yield chat_a, chat_b, history_a, history_b, inputs, ""
+    # if HF_TOKEN and do_save:
+    #     try:
+    #         print("Pushing prompt and completion to the Hub")
+    #         save_inputs_and_outputs(formatted_message, output, generate_kwargs)
+    #     except Exception as e:
+    #         print(e)
+
+    # return [output, output]
+
+
+examples = [
+    "A llama is in my lawn. How do I get rid of him?",
+    "What's the capital city of Brunei?",
+    "How can I sort a list in Python?",
+    "What's the meaning of life?",
+    "How can I write a Java function to generate the nth Fibonacci number?",
+]
+
+
+def regenerate(inputs, model_a, chatbot_a, history_a, model_b, chatbot_b, history_b, temperature, max_new_tokens, top_p, repetition_penalty, do_save):
+    # Do nothing if there's no history
+    if has_no_history(chatbot_a, chatbot_b, history_a, history_b):
+        print("NOTHING")
+        return
+
+    chatbot_a = chatbot_a[:-1]
+    chatbot_b = chatbot_b[:-1]
+    history_a = history_a[:-2]
+    history_b = history_b[:-2]
+
+    for chat_a, chat_b, history_a, history_b, inputs, _ in generate(inputs, model_a, chatbot_a, history_a, model_b, chatbot_b, history_b, temperature, max_new_tokens, top_p, repetition_penalty, do_save):
+        yield chat_a, chat_b, history_a, history_b, inputs, ""
+
+
+def clear_chat():
+    return [], [], [], []
+
+
+def process_example(args):
+    print("process_examples")
+    print(args)
+    for [x, y] in generate(args):
+        pass
+    return [x, y]
+
+
+title = """<h1 align="center">🥊 LLM vs LLM 🏆</h1>"""
+custom_css = """
+#banner-image {
+  display: block;
+  margin-left: auto;
+  margin-right: auto;
+  width: 50%;
+}
+
+.model-dropdown {
+  color: black !important;
+}
+
+#chatbot-a .message {
+ padding: 15px;
+ border-color: #a5b4fc;
+ background-color: #eef2ff;
+}
+
+#chatbot-b .message {
+ padding: 15px;
+ border-color: #fdba74;
+ background-color: #fff7ed;
+}
+
+#chatbot-a .message.bot {
+ padding: 15px;
+ border-color: #e2e8f0;
+    background-color: #f8fafc;
+}
+
+#chatbot-b .message.bot {
+ padding: 15px;
+ border-color: #e2e8f0;
+    background-color: #f8fafc;
+}
+
+#chatbot-a {
+    min-height: 600px;
+}
+
+#chatbot-b {
+    min-height: 600px;
+}
+
+"""
+
+css = share_btn_css + custom_css
+
+with gr.Blocks(theme=theme, analytics_enabled=False, css=css) as demo:
+    gr.HTML(title)
+    gr.Image("llms.png", elem_id="banner-image", show_label=False)
+    gr.Markdown(
+        """
+            Compare two language language models side-by-side.
+            
+            ⚠️ **Data Collection**: by default, we are collecting the prompts entered in this app to further improve and evaluate the model. Do not share any personal or sensitive information while using the app! You can opt out of this data collection by removing the checkbox below:
+      """
+    )
+
+    gr.Dataframe(
+        value=table_data,
+        headers=["Model", "Description", "Source"],
+        row_count=[2, "dynamic"],
+        col_count=[3, "fixed"],
+        datatype=["str", "str", "str"],
+        type="array",
+        show_label=False
+    )
+
+    with gr.Row():
+        with gr.Column():
+            with gr.Box():
+                model_a = gr.Dropdown(model_list, elem_classes="model-dropdown", label="Model A", value=CHATBOT_A_DEFAULT)
+                output_a = gr.Markdown()
+                chatbot_a = gr.Chatbot(label="Model A", elem_id="chatbot-a", show_label=False)
+        with gr.Column():
+            with gr.Box():
+                model_b = gr.Dropdown(model_list, elem_classes="model-dropdown", label="Model B", value=CHATBOT_B_DEFAULT)
+                output_b = gr.Markdown()
+                chatbot_b = gr.Chatbot(label="Model B", elem_id="chatbot-b", show_label=False)
+
+    with gr.Row():
+        with gr.Column(scale=3):
+            do_save = gr.Checkbox(
+                value=True,
+                label="Store data",
+                info="You agree to the storage of your prompt and generated text for research and development purposes:")
+            message = gr.Textbox(placeholder="Enter your message here",
+                                 show_label=False, elem_id="q-input")
+            with gr.Row():
+                send_button = gr.Button("Send", elem_id="send-btn", visible=True)
+                regenerate_button = gr.Button("Regenerate", elem_id="send-btn", visible=True)
+
+                clear_chat_button = gr.Button("Clear chat", elem_id="clear-btn", visible=True)
+
+            with gr.Group(elem_id="share-btn-container"):
+                community_icon = gr.HTML(community_icon_html, visible=True)
+                loading_icon = gr.HTML(loading_icon_html, visible=True)
+                share_button = gr.Button("Share to community", elem_id="share-btn", visible=True)
+            with gr.Row():
+                gr.Examples(
+                    examples=examples,
+                    inputs=[message],
+                    cache_examples=False,
+                    fn=process_example,
+                    outputs=[output_a, output_b],
+                )
+
+        with gr.Column(scale=1):
+            temperature = gr.Slider(
+                label="Temperature",
+                value=0.9,
+                minimum=0.0,
+                maximum=2.0,
+                step=0.1,
+                interactive=True,
+                info="Higher values produce more diverse outputs",
+            )
+            max_new_tokens = gr.Slider(
+                label="Max new tokens",
+                value=256,
+                minimum=0,
+                maximum=512,
+                step=4,
+                interactive=True,
+                info="The maximum numbers of new tokens",
+            )
+            top_p = gr.Slider(
+                label="Top-p (nucleus sampling)",
+                value=0.25,
+                minimum=0.0,
+                maximum=1,
+                step=0.05,
+                interactive=True,
+                info="Higher values sample more low-probability tokens",
+            )
+            repetition_penalty = gr.Slider(
+                label="Repetition penalty",
+                value=1.2,
+                minimum=1.0,
+                maximum=2.0,
+                step=0.05,
+                interactive=True,
+                info="Penalize repeated tokens",
+            )
+
+    history_a = gr.State([])
+    history_b = gr.State([])
+    # To clear out "message" input textbox and use this to regenerate message
+    last_user_message = gr.State("")
+
+    message.submit(generate,
+                   inputs=[message,
+                           model_a,
+                           chatbot_a,
+                           history_a,
+                           model_b,
+                           chatbot_b,
+                           history_b,
+                           temperature,
+                           max_new_tokens,
+                           top_p,
+                           repetition_penalty,
+                           do_save],
+                   outputs=[chatbot_a, chatbot_b, history_a, history_b, last_user_message, message])
+
+    send_button.click(generate,
+                      inputs=[message,
+                              model_a,
+                              chatbot_a,
+                              history_a,
+                              model_b,
+                              chatbot_b,
+                              history_b,
+                              temperature,
+                              max_new_tokens,
+                              top_p,
+                              repetition_penalty,
+                              do_save],
+                      outputs=[chatbot_a, chatbot_b, history_a, history_b, last_user_message, message])
+
+    regenerate_button.click(regenerate, inputs=[last_user_message,
+                                                model_a,
+                                                chatbot_a,
+                                                history_a,
+                                                model_b,
+                                                chatbot_b,
+                                                history_b,
+                                                temperature,
+                                                max_new_tokens,
+                                                top_p,
+                                                repetition_penalty,
+                                                do_save],
+                            outputs=[chatbot_a, chatbot_b, history_a, history_b, message])
+
+    clear_chat_button.click(clear_chat,
+                            outputs=[chatbot_a, chatbot_b, history_a, history_b])
+
+    share_button.click(None, [], [], _js=share_js)
+
+demo.queue(concurrency_count=16).launch(debug=True)

chatty-language-models.py

@@ -0,0 +1,189 @@
+import asyncio
+import os
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import gradio as gr
+import requests
+# from dotenv import load_dotenv
+
+# if Path(".env").is_file():
+#     load_dotenv(".env")
+
+
+TOKEN = "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"
+URL_TO_MODEL = {
+    "https://woyivrd1vhfnxckx.us-east-1.aws.endpoints.huggingface.cloud": "sft",
+    "https://i1qe9e7uv7jzsg8k.us-east-1.aws.endpoints.huggingface.cloud": "rl",
+}
+
+PROMPT_TEMPLATE = "<|system|>\n{system}<|end|>\n<|user|>\n{prompt}<|end|>\n<|assistant|>"
+
+
+def fetch(session, system, text, api_url, temperature, top_p, top_k, max_new_tokens):
+    model = URL_TO_MODEL[api_url]
+    prompt = PROMPT_TEMPLATE.format(system=system, prompt=text)
+    response = session.post(
+        api_url,
+        json={
+            "inputs": prompt,
+            "parameters": {
+                "do_sample": True,
+                "temperature": temperature,
+                "top_p": top_p,
+                "top_k": top_k,
+                "max_new_tokens": max_new_tokens,
+                "eos_token_id": [49155, 32003],
+            },
+        },
+    )
+    if response.status_code != 200:
+        return model, None
+    completion = response.json()[0]["generated_text"]
+    print(f"[MY_LOG] {completion}")
+    if text in completion:
+        completion = completion.lstrip()[len(f"{system}\n{text}\n") :]
+    return model, completion
+
+
+theme = gr.themes.Monochrome(
+    primary_hue="indigo",
+    secondary_hue="blue",
+    neutral_hue="slate",
+    radius_size=gr.themes.sizes.radius_sm,
+    font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"],
+)
+
+
+async def inference(system, text, temperature, top_p, top_k, max_new_tokens):
+    with ThreadPoolExecutor(max_workers=2) as executor:
+        with requests.Session() as session:
+            session.headers = {"Authorization": f"Bearer {TOKEN}"}
+            # Initialize the event loop
+            loop = asyncio.get_event_loop()
+            tasks = [
+                loop.run_in_executor(
+                    executor,
+                    fetch,
+                    *(
+                        session,
+                        system,
+                        text,
+                        url,
+                        temperature,
+                        top_p,
+                        top_k,
+                        max_new_tokens,
+                    ),  # Allows us to pass in multiple arguments to `fetch`
+                )
+                for url in URL_TO_MODEL.keys()
+            ]
+
+            # Initializes the tasks to run and awaits their results
+            responses = [None, None]
+            for model, response in await asyncio.gather(*tasks):
+                print(f"[MY_LOG] response: {model, response}")
+                if response is not None:
+                    if model == "sft":
+                        responses[0] = response
+                    elif model == "rl":
+                        responses[1] = response
+    return responses
+
+
+examples = [
+    # "How many helicopters can a human eat in one sitting?",
+    # "What is an alpaca? How is it different from a llama?",
+    # "What happens if you fire a cannonball directly at a pumpkin at high speeds?",
+    # "Explain the moon landing to a 6 year old in a few sentences.",
+    # "Why aren't birds real?",
+    # "How can I steal from a grocery store without getting caught?",
+    # "Why is it important to eat socks after meditating?",
+    "What date is it today? Use Python to answer the question.",
+    "Write a Python function that replaces all occurrences of the word llama with alpaca",
+    "Write a program to find the nth Fibonacci number using dynamic programming.",
+    "Explain the difference between a function and method in Python.",
+    "What is better, Python or Java?",
+    "Given an array of integers nums and an integer target, return indices of the two numbers such that they add up to target. Write a solution in Python",
+]
+
+
+with gr.Blocks(theme=theme) as demo:
+    gr.Markdown(
+        """<h1><center>Chatty Language Models</center></h1>
+
+        This demo compares two language models trained on a mix of public datasets for instruction fine-tuning and reward modelling. **For internal use only**
+"""
+    )
+    with gr.Row():
+        with gr.Column():
+            with gr.Row():
+                system = gr.Textbox(label="System prompt")
+            with gr.Row():
+                query = gr.Textbox(label="Question")
+            with gr.Row():
+                with gr.Column():
+                    with gr.Row():
+                        temperature = gr.Slider(
+                            label="Temperature",
+                            value=0.2,
+                            minimum=0.0,
+                            maximum=2.0,
+                            step=0.1,
+                            interactive=True,
+                            info="Higher values produce more diverse outputs",
+                        )
+                with gr.Column():
+                    with gr.Row():
+                        top_p = gr.Slider(
+                            label="Top-p (nucleus sampling)",
+                            value=0.9,
+                            minimum=0.0,
+                            maximum=1,
+                            step=0.05,
+                            interactive=True,
+                            info="Higher values sample fewer low-probability tokens",
+                        )
+                with gr.Column():
+                    with gr.Row():
+                        top_k = gr.Slider(
+                            label="Top-k",
+                            value=50,
+                            minimum=0.0,
+                            maximum=100,
+                            step=1,
+                            interactive=True,
+                            info="Sample from a shortlist of top-k tokens",
+                        )
+                with gr.Column():
+                    with gr.Row():
+                        max_new_tokens = gr.Slider(
+                            label="Maximum new tokens",
+                            value=64,
+                            minimum=0,
+                            maximum=2048,
+                            step=5,
+                            interactive=True,
+                            info="The maximum number of new tokens to generate",
+                        )
+            with gr.Row():
+                text_button = gr.Button("Generate answers")
+    with gr.Row():
+        with gr.Column():
+            with gr.Box():
+                gr.Markdown("**Alpaca 7B (baseline)**")
+                baseline_output = gr.Markdown()
+        with gr.Column():
+            with gr.Box():
+                gr.Markdown("**StarChat**")
+                model_output = gr.Markdown()
+    with gr.Row():
+        gr.Examples(examples=examples, inputs=[query])
+
+    text_button.click(
+        inference,
+        inputs=[system, query, temperature, top_p, top_k, max_new_tokens],
+        outputs=[baseline_output, model_output],
+    )
+
+demo.launch()

llm-streaming.py

@@ -0,0 +1,319 @@
+import os
+
+import gradio as gr
+
+from text_generation import Client, InferenceAPIClient
+
+openchat_preprompt = (
+    "\n<human>: Hi!\n<bot>: My name is Bot, model version is 0.15, part of an open-source kit for "
+    "fine-tuning new bots! I was created by Together, LAION, and Ontocord.ai and the open-source "
+    "community. I am not human, not evil and not alive, and thus have no thoughts and feelings, "
+    "but I am programmed to be helpful, polite, honest, and friendly.\n"
+)
+
+
+def get_client(model: str):
+    if model == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
+        return Client(os.getenv("OPENCHAT_API_URL"))
+    return InferenceAPIClient(model, token=os.getenv("HF_TOKEN", None))
+
+
+def get_usernames(model: str):
+    """
+    Returns:
+        (str, str, str, str): pre-prompt, username, bot name, separator
+    """
+    if model in ("OpenAssistant/oasst-sft-1-pythia-12b", "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"):
+        return "", "<|prompter|>", "<|assistant|>", "<|endoftext|>"
+    if model == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
+        return openchat_preprompt, "<human>: ", "<bot>: ", "\n"
+    return "", "User: ", "Assistant: ", "\n"
+
+
+def predict(
+    model: str,
+    inputs: str,
+    typical_p: float,
+    top_p: float,
+    temperature: float,
+    top_k: int,
+    repetition_penalty: float,
+    watermark: bool,
+    chatbot,
+    history,
+):
+    client = get_client(model)
+    preprompt, user_name, assistant_name, sep = get_usernames(model)
+
+    history.append(inputs)
+
+    past = []
+    for data in chatbot:
+        user_data, model_data = data
+
+        if not user_data.startswith(user_name):
+            user_data = user_name + user_data
+        if not model_data.startswith(sep + assistant_name):
+            model_data = sep + assistant_name + model_data
+
+        past.append(user_data + model_data.rstrip() + sep)
+
+    if not inputs.startswith(user_name):
+        inputs = user_name + inputs
+
+    total_inputs = preprompt + "".join(past) + inputs + sep + assistant_name.rstrip()
+
+    partial_words = ""
+
+    if model in ("OpenAssistant/oasst-sft-1-pythia-12b", "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"):
+        iterator = client.generate_stream(
+            total_inputs,
+            typical_p=typical_p,
+            truncate=1000,
+            watermark=watermark,
+            max_new_tokens=500,
+        )
+    else:
+        iterator = client.generate_stream(
+            total_inputs,
+            top_p=top_p if top_p < 1.0 else None,
+            top_k=top_k,
+            truncate=1000,
+            repetition_penalty=repetition_penalty,
+            watermark=watermark,
+            temperature=temperature,
+            max_new_tokens=500,
+            stop_sequences=[user_name.rstrip(), assistant_name.rstrip()],
+        )
+
+    for i, response in enumerate(iterator):
+        if response.token.special:
+            continue
+
+        partial_words = partial_words + response.token.text
+        if partial_words.endswith(user_name.rstrip()):
+            partial_words = partial_words.rstrip(user_name.rstrip())
+        if partial_words.endswith(assistant_name.rstrip()):
+            partial_words = partial_words.rstrip(assistant_name.rstrip())
+
+        if i == 0:
+            history.append(" " + partial_words)
+        elif response.token.text not in user_name:
+            history[-1] = partial_words
+
+        chat = [
+            (history[i].strip(), history[i + 1].strip())
+            for i in range(0, len(history) - 1, 2)
+        ]
+        yield chat, history
+
+
+def reset_textbox():
+    return gr.update(value="")
+
+
+def radio_on_change(
+    value: str,
+    disclaimer,
+    typical_p,
+    top_p,
+    top_k,
+    temperature,
+    repetition_penalty,
+    watermark,
+):
+    if value in ("OpenAssistant/oasst-sft-1-pythia-12b", "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"):
+        typical_p = typical_p.update(value=0.2, visible=True)
+        top_p = top_p.update(visible=False)
+        top_k = top_k.update(visible=False)
+        temperature = temperature.update(visible=False)
+        disclaimer = disclaimer.update(visible=False)
+        repetition_penalty = repetition_penalty.update(visible=False)
+        watermark = watermark.update(False)
+    elif value == "togethercomputer/GPT-NeoXT-Chat-Base-20B":
+        typical_p = typical_p.update(visible=False)
+        top_p = top_p.update(value=0.25, visible=True)
+        top_k = top_k.update(value=50, visible=True)
+        temperature = temperature.update(value=0.6, visible=True)
+        repetition_penalty = repetition_penalty.update(value=1.01, visible=True)
+        watermark = watermark.update(False)
+        disclaimer = disclaimer.update(visible=True)
+    else:
+        typical_p = typical_p.update(visible=False)
+        top_p = top_p.update(value=0.95, visible=True)
+        top_k = top_k.update(value=4, visible=True)
+        temperature = temperature.update(value=0.5, visible=True)
+        repetition_penalty = repetition_penalty.update(value=1.03, visible=True)
+        watermark = watermark.update(True)
+        disclaimer = disclaimer.update(visible=False)
+    return (
+        disclaimer,
+        typical_p,
+        top_p,
+        top_k,
+        temperature,
+        repetition_penalty,
+        watermark,
+    )
+
+
+title = """<h1 align="center">Large Language Model Chat API</h1>"""
+description = """Language models can be conditioned to act like dialogue agents through a conversational prompt that typically takes the form:
+
+```
+User: <utterance>
+Assistant: <utterance>
+User: <utterance>
+Assistant: <utterance>
+...
+```
+
+In this app, you can explore the outputs of multiple LLMs when prompted in this way.
+"""
+
+text_generation_inference = """
+<div align="center">Powered by: <a href=https://github.com/huggingface/text-generation-inference>Text Generation Inference</a></div>
+"""
+
+openchat_disclaimer = """
+<div align="center">Checkout the official <a href=https://huggingface.co/spaces/togethercomputer/OpenChatKit>OpenChatKit feedback app</a> for the full experience.</div>
+"""
+
+with gr.Blocks(
+    css="""#col_container {margin-left: auto; margin-right: auto;}
+                #chatbot {height: 520px; overflow: auto;}"""
+) as demo:
+    gr.HTML(title)
+    gr.Markdown(text_generation_inference, visible=True)
+    with gr.Column(elem_id="col_container"):
+        model = gr.Radio(
+            value="OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
+            choices=[
+                "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5",
+                "OpenAssistant/oasst-sft-1-pythia-12b",
+                # "togethercomputer/GPT-NeoXT-Chat-Base-20B",
+                "google/flan-t5-xxl",
+                "google/flan-ul2",
+                "bigscience/bloom",
+                "bigscience/bloomz",
+                "EleutherAI/gpt-neox-20b",
+            ],
+            label="Model",
+            interactive=True,
+        )
+
+        chatbot = gr.Chatbot(elem_id="chatbot")
+        inputs = gr.Textbox(
+            placeholder="Hi there!", label="Type an input and press Enter"
+        )
+        disclaimer = gr.Markdown(openchat_disclaimer, visible=False)
+        state = gr.State([])
+        b1 = gr.Button()
+
+        with gr.Accordion("Parameters", open=False):
+            typical_p = gr.Slider(
+                minimum=-0,
+                maximum=1.0,
+                value=0.2,
+                step=0.05,
+                interactive=True,
+                label="Typical P mass",
+            )
+            top_p = gr.Slider(
+                minimum=-0,
+                maximum=1.0,
+                value=0.25,
+                step=0.05,
+                interactive=True,
+                label="Top-p (nucleus sampling)",
+                visible=False,
+            )
+            temperature = gr.Slider(
+                minimum=-0,
+                maximum=5.0,
+                value=0.6,
+                step=0.1,
+                interactive=True,
+                label="Temperature",
+                visible=False,
+            )
+            top_k = gr.Slider(
+                minimum=1,
+                maximum=50,
+                value=50,
+                step=1,
+                interactive=True,
+                label="Top-k",
+                visible=False,
+            )
+            repetition_penalty = gr.Slider(
+                minimum=0.1,
+                maximum=3.0,
+                value=1.03,
+                step=0.01,
+                interactive=True,
+                label="Repetition Penalty",
+                visible=False,
+            )
+            watermark = gr.Checkbox(value=False, label="Text watermarking")
+
+    model.change(
+        lambda value: radio_on_change(
+            value,
+            disclaimer,
+            typical_p,
+            top_p,
+            top_k,
+            temperature,
+            repetition_penalty,
+            watermark,
+        ),
+        inputs=model,
+        outputs=[
+            disclaimer,
+            typical_p,
+            top_p,
+            top_k,
+            temperature,
+            repetition_penalty,
+            watermark,
+        ],
+    )
+
+    inputs.submit(
+        predict,
+        [
+            model,
+            inputs,
+            typical_p,
+            top_p,
+            temperature,
+            top_k,
+            repetition_penalty,
+            watermark,
+            chatbot,
+            state,
+        ],
+        [chatbot, state],
+    )
+    b1.click(
+        predict,
+        [
+            model,
+            inputs,
+            typical_p,
+            top_p,
+            temperature,
+            top_k,
+            repetition_penalty,
+            watermark,
+            chatbot,
+            state,
+        ],
+        [chatbot, state],
+    )
+    b1.click(reset_textbox, [], [inputs])
+    inputs.submit(reset_textbox, [], [inputs])
+
+    gr.Markdown(description)
+    demo.queue(concurrency_count=16).launch(debug=True)

share_btn.py

@@ -76,7 +76,7 @@ ${outputTxt}`;
         .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`)
         .join('&');
 
-	window.open(`https://huggingface.co/spaces/trl-lib/stack-llama/discussions/new?${paramsStr}`, '_blank');
+	window.open(`https://huggingface.co/spaces/HuggingFaceH4/star-chat-demo/discussions/new?${paramsStr}`, '_blank');
 
     shareBtnEl.style.removeProperty('pointer-events');
     shareIconEl.style.removeProperty('display');

stack-llama.py

@@ -0,0 +1,186 @@
+import json
+import os
+import shutil
+
+import gradio as gr
+from huggingface_hub import Repository
+from text_generation import Client
+
+from share_btn import community_icon_html, loading_icon_html, share_js, share_btn_css
+
+HF_TOKEN = os.environ.get("H4_TOKEN", None)
+# HF_TOKEN = os.environ.get("TRL_TOKEN", None)
+API_URL = os.environ.get("API_URL")
+
+theme = gr.themes.Monochrome(
+    primary_hue="indigo",
+    secondary_hue="blue",
+    neutral_hue="slate",
+    radius_size=gr.themes.sizes.radius_sm,
+    font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"],
+)
+if HF_TOKEN:
+    try:
+        shutil.rmtree("./data/")
+    except:
+        pass
+
+    repo = Repository(
+        local_dir="./data/", clone_from="h4/star-chat-prompts", use_auth_token=HF_TOKEN, repo_type="dataset"
+    )
+    repo.git_pull()
+
+client = Client(
+    API_URL,
+    headers={"Authorization": f"Bearer {HF_TOKEN}"},
+)
+
+PROMPT_TEMPLATE = """Question: {prompt}\n\nAnswer:"""
+
+
+def save_inputs_and_outputs(inputs, outputs, generate_kwargs):
+    with open(os.path.join("data", "prompts.jsonl"), "a") as f:
+        json.dump({"inputs": inputs, "outputs": outputs, "generate_kwargs": generate_kwargs}, f, ensure_ascii=False)
+        f.write("\n")
+        commit_url = repo.push_to_hub()
+
+
+def generate(instruction, temperature=0.9, max_new_tokens=256, top_p=0.95, repetition_penalty=1.0,  do_save=True):
+    formatted_instruction = PROMPT_TEMPLATE.format(prompt=instruction)
+
+    temperature = float(temperature)
+    if temperature < 1e-2:
+        temperature = 1e-2
+    top_p = float(top_p)
+
+    generate_kwargs = dict(
+        temperature=temperature,
+        max_new_tokens=max_new_tokens,
+        top_p=top_p,
+        repetition_penalty=repetition_penalty,
+        do_sample=True,
+        truncate=999,
+        seed=42,
+        stop_sequences=["</s>"],
+    )
+
+    stream = client.generate_stream(
+        formatted_instruction,
+        **generate_kwargs,
+    )
+
+    output = ""
+    for response in stream:
+        output += response.token.text
+        yield output
+    if HF_TOKEN and do_save:
+        try:
+            print("Pushing prompt and completion to the Hub")
+            save_inputs_and_outputs(formatted_instruction, output, generate_kwargs)
+        except Exception as e:
+            print(e)
+            
+    return output
+
+
+examples = [
+    "A llama is in my lawn. How do I get rid of him?",
+    "What are the various algorithms to sort a list?",
+    "How can I sort a list in Python?",
+    "How do I ask a question in StackOverflow?",
+    "How to beat a Hitmonlee in a Pokemon battle?",
+    "How can I write a Java function to generate the nth Fibonacci number?",
+]
+
+
+def process_example(args):
+    for x in generate(args):
+        pass
+    return x
+
+css = ".generating {visibility: hidden}" + share_btn_css
+
+with gr.Blocks(theme=theme, analytics_enabled=False, css=css) as demo:
+    with gr.Column():
+        gr.Markdown(
+            """![](https://huggingface.co/spaces/trl-lib/stack-llama/resolve/main/stackllama_logo.png)
+
+
+            StackLLaMa is a 7 billion parameter language model based on [Meta's LLaMA model](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/) that has been trained on pairs of questions and answers from [Stack Exchange](https://stackexchange.com) using Reinforcement Learning from Human Feedback (RLHF) with the [TRL library](https://github.com/lvwerra/trl). For more details, check out our [blog post](https://huggingface.co/blog/stackllama).
+
+            Type in the box below and click the button to generate answers to your most pressing questions!
+
+            ⚠️ **Intended Use**: this app and its [supporting model](https://huggingface.co/trl-lib/llama-7b-se-rl-peft) are provided as educational tools to explain RLHF with the TRL library; not to serve as replacement for human expertise. For more details on the model's limitations in terms of factuality and biases, see the [model card.](https://huggingface.co/trl-lib/llama-7b-se-rl-peft#intended-uses--limitations)
+            
+            ⚠️ **Data Collection**: by default, we are collecting the prompts entered in this app to further improve and evaluate the model. Do not share any personal or sensitive information while using the app! You can opt out of this data collection by removing the checkbox below:
+      """
+        )
+        with gr.Row():
+            with gr.Column(scale=3):
+                do_save = gr.Checkbox(
+                        value=True,
+                        label="Store data",
+                        info="You agree to the storage of your prompt and generated text for research and development purposes:")
+                instruction = gr.Textbox(placeholder="Enter your question here", label="Question", elem_id="q-input")
+
+                
+                with gr.Box():
+                    gr.Markdown("**Answer**")
+                    output = gr.Markdown(elem_id="q-output")
+                submit = gr.Button("Generate", variant="primary")
+                with gr.Group(elem_id="share-btn-container"):
+                    community_icon = gr.HTML(community_icon_html, visible=True)
+                    loading_icon = gr.HTML(loading_icon_html, visible=True)
+                    share_button = gr.Button("Share to community", elem_id="share-btn", visible=True)
+                gr.Examples(
+                    examples=examples,
+                    inputs=[instruction],
+                    cache_examples=False,
+                    fn=process_example,
+                    outputs=[output],
+                )
+
+            with gr.Column(scale=1):
+                
+                temperature = gr.Slider(
+                    label="Temperature",
+                    value=0.9,
+                    minimum=0.0,
+                    maximum=2.0,
+                    step=0.1,
+                    interactive=True,
+                    info="Higher values produce more diverse outputs",
+                )
+                max_new_tokens = gr.Slider(
+                    label="Max new tokens",
+                    value=256,
+                    minimum=0,
+                    maximum=512,
+                    step=4,
+                    interactive=True,
+                    info="The maximum numbers of new tokens",
+                )
+                top_p = gr.Slider(
+                    label="Top-p (nucleus sampling)",
+                    value=0.90,
+                    minimum=0.0,
+                    maximum=1,
+                    step=0.05,
+                    interactive=True,
+                    info="Higher values sample more low-probability tokens",
+                )
+                repetition_penalty = gr.Slider(
+                    label="Repetition penalty",
+                    value=1.2,
+                    minimum=1.0,
+                    maximum=2.0,
+                    step=0.05,
+                    interactive=True,
+                    info="Penalize repeated tokens",
+                )
+
+    submit.click(generate, inputs=[instruction, temperature, max_new_tokens, top_p, repetition_penalty, do_save], outputs=[output])
+    instruction.submit(generate, inputs=[instruction, temperature, max_new_tokens, top_p, repetition_penalty], outputs=[output])
+    share_button.click(None, [], [], _js=share_js)
+
+demo.queue(concurrency_count=16).launch(debug=True)