Update app.py

changed code to use gr interface and vicuna 13B

app.py

@@ -1,90 +1,122 @@
 # imports
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
-import requests
-import random
-import torch
-import os
+from huggingface_hub import InferenceClient
 
 # lists for random gen, not the best format here but it runs fine
 book_genres = ["Adventure", "Romance","Mystery", "Science Fiction","Fantasy","Thriller","Horror","Historical Fiction","Biography","Autobiography","Self-Help","Non-Fiction","Science","Cooking","Travel","Dystopian","Young Adult","Children's","Poetry","Classic","Graphic Novel","Humor","Crime","Western","Memoir","Religion","Psychology","Philosophy","Business","Finance","Parenting","Health","Fitness","Art","Music","Sports","Politics","Education","Technology","Science Fiction Fantasy","Steampunk","Drama","Historical Non-Fiction","Biographical Fiction","Mythology","Anthology","Short Stories","Essays","Fairy Tales","Magic Realism","True Crime","Satire","Romantic Suspense","Paranormal","Urban Fantasy","War","Epic Fantasy","Contemporary Fiction","Legal Thriller","Espionage","Post-Apocalyptic","Time Travel","Cultural","Medical","Environmental","Artificial Intelligence","Cyberpunk","Space Opera","Alternate History","Historical Romance","Science Fiction Romance","Young Adult Fantasy","Middle Grade","Adventure Fantasy","Superhero","Graphic Memoir","Travel Memoir","Political Thriller","Economic","Psychological Thriller","Nature","True Adventure","Historical Mystery","Social Science","Science Biography","Space Exploration","Pop Culture","Art History","Culinary","Nature Writing","Family Drama","Classic Literature","Cultural History","Political Science","Economics","Essays and Criticism","Art Criticism","Criminal Justice","Historical Biography","Personal Development","Cookbook","Fashion","Crafts and Hobbies","Memoir","Essays","Graphic Non-Fiction", "Fantasy Romance"]
 book_themes = ["Love and Relationships","Friendship","Family","Coming of Age","Identity and Self-discovery","Adventure and Exploration","Mystery and Intrigue","Science and Technology","Fantasy Worlds","Historical Events","War and Conflict","Survival","Good vs. Evil","Justice and Morality","Revenge","Betrayal","Hope and Resilience","Isolation and Loneliness","Social Justice","Environmental Conservation","Political Corruption","Human Rights","Dystopia","Utopia","Alien Encounters","Time Travel","Art and Creativity","Death and Mortality","Cultural Identity","Personal Growth","Addiction","Education and Knowledge","Freedom and Liberation","Equality and Inequality","Society and Class","Legacy and Inheritance","Religion and Spirituality","Grief and Loss","Ambition","Transformation","Humor and Satire","Survival of the Fittest","Dreams and Aspirations","Change and Adaptation","Forgiveness","Nature and the Environment","Exploration of the Unknown","Conflict Resolution","Fate and Destiny","Artificial Intelligence","Cybersecurity","Space Exploration","Parallel Universes","Economic Struggles","Social Media and Technology","Innovation and Invention","Psychological Thrills","Philosophical Contemplation","Ancient Mythology","Modern Mythology","Epic Journeys","The Power of Imagination","Unrequited Love","Secrets and Hidden Truths","Warriors and Heroes","Surviving Adversity","Dreams and Nightmares","Rivalry and Competition","Alien Worlds","Conspiracy","Apocalyptic Scenarios","Conformity vs. Individuality","Legacy and Heritage","Nature vs. Nurture","Moral Dilemmas","Adventure and Discovery","Journey of Self-Discovery","Unlikely Friendships","Struggle for Power","Exploration of Fear","The Supernatural","Cultural Clashes","Identity Crisis","The Quest for Knowledge","The Human Condition","Hidden Agendas","Escapism","The Pursuit of Happiness","Redemption","Rebellion","Feminism and Gender Issues","Exploration of Dreams","Innocence vs. Experience","Chaos and Order","Exploration of Evil"]
 writing_tones = ["Formal","Informal","Humorous","Serious","Sarcastic","Satirical","Melancholic","Optimistic","Pessimistic","Cynical","Hopeful","Lighthearted","Dark","Gothic","Whimsical","Mysterious","Eerie","Solemn","Playful","Thoughtful","Reflective","Ironic","Sensual","Nostalgic","Surreal","Dreamy","Awe-Inspiring","Introspective","Confessional","Dramatic","Exuberant","Melodramatic","Hypnotic","Inspirational","Tongue-in-Cheek","Witty","Calm","Passionate","Detached","Frightening","Intense","Calm","Suspenseful","Brave","Desperate","Eloquent","Vivid","Casual","Whispering","Eloquent","Bitter","Tragic","Pensive","Frenzied","Melodious","Resolute","Soothing","Brisk","Lyrical","Objective","Factual","Contemplative","Sardonic","Sympathetic","Objective","Sincere","Wistful","Stoic","Empathetic","Matter-of-fact","Sentimental","Sharp","Understated","Exaggerated","Casual","Bombastic","Poetic","Charming","Apologetic","Defensive","Confrontational","Inquisitive","Candid","Reverent","Matter-of-fact","Amusing","Enthusiastic","Questioning","Reproachful","Hopeless","Despondent","Wry","Sulking","Serene","Detached","Confident","Steadfast","Foolish","Impassioned","Indignant","Self-Deprecating","Wandering","Inspiring","Bewildered"]
 
-model_id = "Nexusflow/NexusRaven-V2-13B"
 
-# API endpoint, idk if I can use it or need to change the format to use AutoModelForCausualLM
-API_ENDPOINT = "https://api-inference.huggingface.co/models/Nexusflow/NexusRaven-V2-13B"  # Replace with the model's API endpoint
-
-# Load the tokenizer
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-API_KEY = HF_TOKEN = os.environ.get("API_KEY")
-
-#main function
-def generate_novel_title_and_summary(genres, moods, themes, temperature=1.5, max_length=512, context_length=1024):
+client = InferenceClient(
+    "mistralai/Mixtral-8x7B-Instruct-v0.1"
+)
 
+def format_prompt(message, history, genres, moods, themes):
     if not genres:
         genres = ", ".join(random.sample(book_genres, random.randint(3, 5)))
     if not moods:       
         moods = ", ".join(random.sample(writing_tones, random.randint(3, 5)))
     if not themes:
         themes = ", ".join(random.sample(book_themes, random.randint(3, 5)))
-
+    
     prompt = f"Write a simple novel title and interesting summary for a book that falls under the following \n genres: {genres}, \n themes: {themes}, \n and moods: {moods}. \n Title:"
-
-    # Generate a new random seed for each request
-    random_seed = random.randint(1, 1000000)
-
-    # Set the random seed for PyTorch
-    torch.manual_seed(random_seed)
-
-    # Prepare the data for the Hugging Face API
-    data = {
-        "inputs": prompt,
-        "options": {
-            "temperature": temperature,
-            "max_length": max_length,
-            "context_length": context_length,
-            "do_sample": True,
-            "use_cache": False
-        }
-    }
-
-    headers = {"Authorization": f"Bearer {API_KEY}"}
-
-    try:
-        # Make the API request
-        response = requests.post(API_ENDPOINT, json=data, headers=headers)
-
-        if response.status_code == 200:
-            result = response.json()
-            generated_text = result[0].get("generated_text", "")
-            return generated_text
-
-        else:
-            return f"Error: {response.status_code} - Unable to generate text."
-
-    except Exception as e:
-        return f"Error: {str(e)} - An error occurred while generating text."
+    for user_prompt, bot_response in history:
+        prompt += f"[INST] {user_prompt} [/INST]"
+        prompt += f" {bot_response}</s> "
+        prompt += f"[INST] {message} [/INST]"
+    return prompt
+
+
+
+def generate(genres, moods, themes, history,  system_prompt, temperature=1.25, max_new_tokens=256, top_p=0.95, repetition_penalty=1.15,):
+ 
+    temperature = float(temperature)
+    if temperature < 1e-2:
+        temperature = 1e-2
+    top_p = float(top_p)
+
+    generate_kwargs = dict(
+        temperature=temperature,
+        max_new_tokens=max_new_tokens,
+        top_p=top_p,
+        repetition_penalty=repetition_penalty,
+        do_sample=True,
+        seed=42,
+    )
+
+    formatted_prompt = format_prompt(f"{system_prompt}, '' ", history, genres, moods, themes)
+    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+    output = ""
+
+    for response in stream:
+        output += response.token.text
+        yield output
+    return output
+
+
+additional_inputs=[
+    gr.Textbox(
+        label="System Prompt",
+        max_lines=1,
+        interactive=True,
+    ),
+    gr.Slider(
+        label="Temperature",
+        value=0.9,
+        minimum=0.0,
+        maximum=1.0,
+        step=0.05,
+        interactive=True,
+        info="Higher values produce more diverse outputs",
+    ),
+    gr.Slider(
+        label="Max new tokens",
+        value=256,
+        minimum=0,
+        maximum=1048,
+        step=64,
+        interactive=True,
+        info="The maximum numbers of new tokens",
+    ),
+    gr.Slider(
+        label="Top-p (nucleus sampling)",
+        value=0.90,
+        minimum=0.0,
+        maximum=1,
+        step=0.05,
+        interactive=True,
+        info="Higher values sample more low-probability tokens",
+    ),
+    gr.Slider(
+        label="Repetition penalty",
+        value=1.2,
+        minimum=1.0,
+        maximum=2.0,
+        step=0.05,
+        interactive=True,
+        info="Penalize repeated tokens",
+    )
+]
 
 def launch_interface():
   iface = gr.Interface(
-      fn=generate_novel_title_and_summary,
+      fn=generate,
       inputs=[
           gr.Textbox("", label="Book Genres (comma-separated, or leave blank!)"),
           gr.Textbox("", label="Book Themes (comma-separated, or leave blank!)"),
           gr.Textbox("", label="Writing Tone (comma-separated, or leave blank!)"),
           gr.Slider(0.1, 10.0, 1.3, label="Temperature (Creativity)"),
-          gr.Slider(32, 512, 300, label="Max Token Length (Length of output)"),
-          gr.Slider(32, 1024, 256, label="Context Length (Memory)")
       ],
-      outputs="text",
-      live=False,
-      title="Novel Title and Summary Generator",
-      theme='ParityError/Interstellar',
+    additional_inputs=additional_inputs,
+    outputs="text",
+    live=False,
+    title="Novel Title and Summary Generator",
+    layout = 'panel'
+    theme='ParityError/Interstellar',
   )
 
-  iface.launch(debug=True)
+    iface.launch(show_api=False)
 
 if __name__ == "__main__":
     launch_interface()