Update app.py

Update with llama_cpp

app.py

@@ -1,31 +1,69 @@
 import streamlit as st
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from langchain_core.prompts import ChatPromptTemplate
-
-# Load the model and tokenizer
-tokenizer = AutoTokenizer.from_pretrained("unsloth/Llama-3.2-3B-Instruct")
-model = AutoModelForCausalLM.from_pretrained("unsloth/Llama-3.2-3B-Instruct")
-
-st.title("Unsloth Llama-3.2-3B-Instruct Text Generation")
-st.write("Enter a prompt and generate text using the Unsloth Llama 3.2 3B model.")
-
-prompt = """
-    You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question.
-    If you don't know the answer, just say that you don't know.
-    Answer in bullet points. Make sure your answer is relevant to the question and it is answered from the context only.
-    Question: {question} 
-    Context: {context} 
-    Answer:
-"""
-
-prompt = ChatPromptTemplate.from_template(prompt)
-
-with st.form("llm-form"):
-    user_input = st.text_area("Enter your question or statement:")
-    submit = st.form_submit_button("Submit")
-
-    if submit:
-        inputs = tokenizer(user_input, return_tensors="pt")
-        outputs = model.generate(inputs["input_ids"], max_length=200)
-        generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
-        st.write(generated_text)
+from llama_cpp import Llama
+
+if 'llm' not in st.session_state:
+    st.session_state.llm = Llama.from_pretrained(
+        repo_id="bartowski/Llama-3.2-3B-Instruct-GGUF",
+        filename="Llama-3.2-3B-Instruct-Q8_0.gguf",  
+        verbose=True,
+        n_ctx=32768,
+        n_threads=2,
+        chat_format="chatml"
+    )
+
+# Define the function to get responses from the model
+def respond(message, history):
+    messages = []
+
+    for user_message, assistant_message in history:
+        if user_message:
+            messages.append({"role": "user", "content": user_message})
+        if assistant_message:
+            messages.append({"role": "assistant", "content": assistant_message})
+
+    messages.append({"role": "user", "content": message})
+
+    response = ""
+    # Stream the response from the model
+    response_stream = st.session_state.llm.create_chat_completion(
+        messages=messages,
+        stream=True,
+        max_tokens=512,  # Use a default value for simplicity
+        temperature=0.7,  # Use a default value for simplicity
+        top_p=0.95  # Use a default value for simplicity
+    )
+
+    # Collect the response chunks
+    for chunk in response_stream:
+        if len(chunk['choices'][0]["delta"]) != 0 and "content" in chunk['choices'][0]["delta"]:
+            response += chunk['choices'][0]["delta"]["content"]
+
+    return response  # Return the full response
+
+# Streamlit UI
+st.title("Simple Chatbot")
+st.write("### Interact with the chatbot!")
+
+# User input field
+user_message = st.text_area("Your Message:", "")
+
+if "chat_history" not in st.session_state:
+    st.session_state['chat_history'] = []
+
+# Button to send the message
+if st.button("Send"):
+    if user_message:  # Check if user has entered a message
+        # Get the response from the model
+        response = respond(user_message, st.session_state['chat_history'])
+
+        # Add user message and model response to history
+        st.session_state['chat_history'].append((user_message, response))
+
+        # Clear the input field after sending
+        user_message = ""  # Reset user_message to clear input
+
+st.write("## Chat History")
+for user_msg, assistant_msg in reversed(st.session_state['chat_history']):
+    st.write(f"**🧑 User**: {user_msg}")
+    st.write(f"**🧠 Assistant**: {assistant_msg}")
+    st.write("---")