Create app

.env.sample

@@ -0,0 +1,5 @@
+# !!! DO NOT UPDATE THIS FILE DIRECTLY. MAKE A COPY AND RENAME IT `.env` TO PROCEED !!! #
+HF_LLM_ENDPOINT="YOUR_LLM_ENDPOINT_URL_HERE"
+HF_EMBED_ENDPOINT="YOUR_EMBED_MODEL_ENDPOINT_URL_HERE"
+HF_TOKEN="YOUR_HF_TOKEN_HERE"
+# !!! DO NOT UPDATE THIS FILE DIRECTLY. MAKE A COPY AND RENAME IT `.env` TO PROCEED !!! #

.gitignore

@@ -0,0 +1,6 @@
+.env
+__pycache__/
+.chainlit
+*.faiss
+*.pkl
+.files

Dockerfile

@@ -0,0 +1,29 @@
+# Get a distribution that has uv already installed
+FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim
+
+# Add user - this is the user that will run the app
+# If you do not set user, the app will run as root (undesirable)
+RUN useradd -m -u 1000 user
+USER user
+
+# Set the home directory and path
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH        
+
+ENV UVICORN_WS_PROTOCOL=websockets
+
+# Set the working directory
+WORKDIR $HOME/app
+
+# Copy the app to the container
+COPY --chown=user . $HOME/app
+
+# Install the dependencies
+# RUN uv sync --frozen
+RUN uv sync
+
+# Expose the port
+EXPOSE 7860
+
+# Run the app
+CMD ["uv", "run", "chainlit", "run", "solution_app.py", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,11 +1,8 @@
 ---
-title: Open Source Embeddings
-emoji: 🌖
-colorFrom: blue
+title: Demo For Session 15
+emoji: 🐢
+colorFrom: yellow
 colorTo: red
 sdk: docker
 pinned: false
-short_description: A chainlit app using open source embeddings
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---

app.py

@@ -0,0 +1,241 @@
+import os
+import chainlit as cl
+from dotenv import load_dotenv
+from operator import itemgetter
+from langchain_huggingface import HuggingFaceEndpoint
+from langchain_community.document_loaders import TextLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEndpointEmbeddings
+from langchain_core.prompts import PromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnablePassthrough
+from langchain.schema.runnable.config import RunnableConfig
+from tqdm.asyncio import tqdm_asyncio
+import asyncio
+from tqdm.asyncio import tqdm
+
+# GLOBAL SCOPE - ENTIRE APPLICATION HAS ACCESS TO VALUES SET IN THIS SCOPE #
+# ---- ENV VARIABLES ---- #
+"""
+This function will load our environment file (.env) if it is present.
+
+NOTE: Make sure that .env is in your .gitignore file - it is by default, but please ensure it remains there.
+"""
+load_dotenv()
+
+"""
+We will load our environment variables here.
+"""
+HF_LLM_ENDPOINT = os.environ["HF_LLM_ENDPOINT"]
+HF_EMBED_ENDPOINT = os.environ["HF_EMBED_ENDPOINT"]
+HF_TOKEN = os.environ["HF_TOKEN"]
+
+# ---- GLOBAL DECLARATIONS ---- #
+
+# -- RETRIEVAL -- #
+"""
+1. Load Documents from Text File
+2. Split Documents into Chunks
+3. Load HuggingFace Embeddings (remember to use the URL we set above)
+4. Index Files if they do not exist, otherwise load the vectorstore
+"""
+# 1. CREATE TEXT LOADER AND LOAD DOCUMENTS
+# NOTE: PAY ATTENTION TO THE PATH THEY ARE IN.
+text_loader = TextLoader("./data/paul_graham_essays.txt")
+documents = text_loader.load()
+
+# 2. CREATE TEXT SPLITTER AND SPLIT DOCUMENTS
+text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size=1000, chunk_overlap=30)
+split_documents = text_splitter.split_documents(documents)
+len(split_documents)
+
+# 3. LOAD HUGGINGFACE EMBEDDINGS
+hf_embeddings = HuggingFaceEndpointEmbeddings(
+    model=HF_EMBED_ENDPOINT,
+    task="feature-extraction",
+    huggingfacehub_api_token=HF_TOKEN,
+)
+
+
+async def add_documents_async(vectorstore, documents):
+    await vectorstore.aadd_documents(documents)
+
+
+async def process_batch(vectorstore, batch, is_first_batch, pbar):
+    if is_first_batch:
+        result = await FAISS.afrom_documents(batch, hf_embeddings)
+    else:
+        await add_documents_async(vectorstore, batch)
+        result = vectorstore
+    pbar.update(len(batch))
+    return result
+
+
+async def main():
+    print("Indexing Files")
+
+    vectorstore = None
+    batch_size = 32
+
+    batches = [split_documents[i:i+batch_size]
+               for i in range(0, len(split_documents), batch_size)]
+
+    async def process_all_batches():
+        nonlocal vectorstore
+        tasks = []
+        pbars = []
+
+        for i, batch in enumerate(batches):
+            pbar = tqdm(total=len(batch),
+                        desc=f"Batch {i+1}/{len(batches)}", position=i)
+            pbars.append(pbar)
+
+            if i == 0:
+                vectorstore = await process_batch(None, batch, True, pbar)
+            else:
+                tasks.append(process_batch(vectorstore, batch, False, pbar))
+
+        if tasks:
+            await asyncio.gather(*tasks)
+
+        for pbar in pbars:
+            pbar.close()
+
+    await process_all_batches()
+
+    hf_retriever = vectorstore.as_retriever()
+    print("\nIndexing complete. Vectorstore is ready for use.")
+    return hf_retriever
+
+
+async def run():
+    retriever = await main()
+    return retriever
+
+hf_retriever = asyncio.run(run())
+
+# -- AUGMENTED -- #
+"""
+1. Define a String Template
+2. Create a Prompt Template from the String Template
+"""
+# 1. DEFINE STRING TEMPLATE
+RAG_PROMPT_TEMPLATE = """\
+<|start_header_id|>system<|end_header_id|>
+You are a helpful assistant. You answer user questions based on provided context. If you can't answer the question with the provided context, say you don't know.<|eot_id|>
+
+<|start_header_id|>user<|end_header_id|>
+User Query:
+{query}
+
+Context:
+{context}<|eot_id|>
+
+<|start_header_id|>assistant<|end_header_id|>
+"""
+
+# 2. CREATE PROMPT TEMPLATE
+rag_prompt = PromptTemplate.from_template(RAG_PROMPT_TEMPLATE)
+
+# -- GENERATION -- #
+"""
+1. Create a HuggingFaceEndpoint for the LLM
+"""
+# 1. CREATE HUGGINGFACE ENDPOINT FOR LLM
+hf_llm = HuggingFaceEndpoint(
+    endpoint_url=f"{HF_LLM_ENDPOINT}",
+    task="text-generation",
+    max_new_tokens=512,
+    top_k=10,
+    top_p=0.95,
+    typical_p=0.95,
+    temperature=0.01,
+    repetition_penalty=1.03,
+    stop_sequences=["<|eot_id|>"]
+)
+
+
+@cl.author_rename
+def rename(original_author: str):
+    """
+    This function can be used to rename the 'author' of a message.
+
+    In this case, we're overriding the 'Assistant' author to be 'Paul Graham Essay Bot'.
+    """
+    rename_dict = {
+        "Assistant": "Paul Graham Essay Bot"
+    }
+    return rename_dict.get(original_author, original_author)
+
+
+@cl.on_chat_start
+async def start_chat():
+    """
+    This function will be called at the start of every user session.
+
+    We will build our LCEL RAG chain here, and store it in the user session.
+
+    The user session is a dictionary that is unique to each user session, and is stored in the memory of the server.
+    """
+
+    # Store the retriever in the user session
+    cl.user_session.set("hf_retriever", hf_retriever)
+
+    # BUILD LCEL RAG CHAIN WITH RETRIEVAL
+    async def get_context(query_dict):
+        docs = await hf_retriever.aget_relevant_documents(query_dict["query"])
+        return {"context": "\n\n".join([doc.page_content for doc in docs]), "query": query_dict["query"]}
+
+    # Post-processing to clean up any token markers
+    def clean_output(text):
+        return text.replace("<|eot_id|>", "").strip()
+
+    # Store individual components
+    cl.user_session.set("get_context", get_context)
+    cl.user_session.set("rag_prompt", rag_prompt)
+    cl.user_session.set("llm", hf_llm)
+
+    # Also store the full chain for non-streaming use cases
+    lcel_rag_chain = (
+        get_context
+        | rag_prompt
+        | hf_llm
+        | StrOutputParser()
+        | clean_output
+    )
+
+    cl.user_session.set("lcel_rag_chain", lcel_rag_chain)
+
+
+@cl.on_message
+async def main(message: cl.Message):
+    """
+    This function will be called every time a message is recieved from a session.
+
+    We will use the LCEL RAG chain to generate a response to the user query.
+
+    The LCEL RAG chain is stored in the user session, and is unique to each user session - this is why we can access it here.
+    """
+    # Get the chain without the clean_output step
+    get_context = cl.user_session.get("get_context")
+    rag_prompt_template = cl.user_session.get("rag_prompt")
+    llm = cl.user_session.get("llm")
+
+    # Create a streaming chain
+    streaming_chain = get_context | rag_prompt_template | llm
+
+    msg = cl.Message(content="")
+
+    # Stream and clean tokens on the fly
+    async for chunk in streaming_chain.astream(
+        {"query": message.content},
+        config=RunnableConfig(callbacks=[cl.LangchainCallbackHandler()]),
+    ):
+        # Clean the chunk before streaming it
+        cleaned_chunk = chunk.replace("<|eot_id|>", "")
+        if cleaned_chunk.strip():  # Only stream non-empty chunks
+            await msg.stream_token(cleaned_chunk)
+
+    await msg.send()

chainlit.md

@@ -0,0 +1 @@
+# FILL OUT YOUR CHAINLIT MD HERE WITH A DESCRIPTION OF YOUR APPLICATION

pyproject.toml

@@ -0,0 +1,22 @@
+[project]
+name = "15-app"
+version = "0.1.0"
+description = "Session 15 - Open Source Endpoints"
+readme = "README.md"
+requires-python = ">=3.09"
+dependencies = [
+    "asyncio===3.4.3",
+    "chainlit==2.2.1",
+    "huggingface-hub==0.27.0",
+    "langchain-huggingface==0.1.2",
+    "langchain==0.3.19",
+    "langchain-community==0.3.18",
+    "langsmith==0.3.11",
+    "python-dotenv==1.0.1",
+    "tqdm==4.67.1",
+    "langchain-openai==0.3.7",
+    "langchain-text-splitters==0.3.6",
+    "jupyter>=1.1.1",
+    "faiss-cpu>=1.10.0",
+    "websockets>=15.0",
+]

solution_app.py

@@ -0,0 +1,190 @@
+import os
+import chainlit as cl
+from dotenv import load_dotenv
+from operator import itemgetter
+from langchain_huggingface import HuggingFaceEndpoint
+from langchain_community.document_loaders import TextLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEndpointEmbeddings
+from langchain_core.prompts import PromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnablePassthrough
+from langchain.schema.runnable.config import RunnableConfig
+from tqdm.asyncio import tqdm_asyncio
+import asyncio
+from tqdm.asyncio import tqdm
+
+# GLOBAL SCOPE - ENTIRE APPLICATION HAS ACCESS TO VALUES SET IN THIS SCOPE #
+# ---- ENV VARIABLES ---- # 
+"""
+This function will load our environment file (.env) if it is present.
+
+NOTE: Make sure that .env is in your .gitignore file - it is by default, but please ensure it remains there.
+"""
+load_dotenv()
+
+"""
+We will load our environment variables here.
+"""
+HF_LLM_ENDPOINT = os.environ["HF_LLM_ENDPOINT"]
+HF_EMBED_ENDPOINT = os.environ["HF_EMBED_ENDPOINT"]
+HF_TOKEN = os.environ["HF_TOKEN"]
+
+# ---- GLOBAL DECLARATIONS ---- #
+
+# -- RETRIEVAL -- #
+"""
+1. Load Documents from Text File
+2. Split Documents into Chunks
+3. Load HuggingFace Embeddings (remember to use the URL we set above)
+4. Index Files if they do not exist, otherwise load the vectorstore
+"""
+document_loader = TextLoader("./data/paul_graham_essays.txt")
+documents = document_loader.load()
+
+text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=30)
+split_documents = text_splitter.split_documents(documents)
+
+hf_embeddings = HuggingFaceEndpointEmbeddings(
+    model=HF_EMBED_ENDPOINT,
+    task="feature-extraction",
+    huggingfacehub_api_token=HF_TOKEN,
+)
+
+async def add_documents_async(vectorstore, documents):
+    await vectorstore.aadd_documents(documents)
+
+async def process_batch(vectorstore, batch, is_first_batch, pbar):
+    if is_first_batch:
+        result = await FAISS.afrom_documents(batch, hf_embeddings)
+    else:
+        await add_documents_async(vectorstore, batch)
+        result = vectorstore
+    pbar.update(len(batch))
+    return result
+
+async def main():
+    print("Indexing Files")
+    
+    vectorstore = None
+    batch_size = 32
+    
+    batches = [split_documents[i:i+batch_size] for i in range(0, len(split_documents), batch_size)]
+    
+    async def process_all_batches():
+        nonlocal vectorstore
+        tasks = []
+        pbars = []
+        
+        for i, batch in enumerate(batches):
+            pbar = tqdm(total=len(batch), desc=f"Batch {i+1}/{len(batches)}", position=i)
+            pbars.append(pbar)
+            
+            if i == 0:
+                vectorstore = await process_batch(None, batch, True, pbar)
+            else:
+                tasks.append(process_batch(vectorstore, batch, False, pbar))
+        
+        if tasks:
+            await asyncio.gather(*tasks)
+        
+        for pbar in pbars:
+            pbar.close()
+    
+    await process_all_batches()
+    
+    hf_retriever = vectorstore.as_retriever()
+    print("\nIndexing complete. Vectorstore is ready for use.")
+    return hf_retriever
+
+async def run():
+    retriever = await main()
+    return retriever
+
+hf_retriever = asyncio.run(run())
+
+# -- AUGMENTED -- #
+"""
+1. Define a String Template
+2. Create a Prompt Template from the String Template
+"""
+RAG_PROMPT_TEMPLATE = """\
+<|start_header_id|>system<|end_header_id|>
+You are a helpful assistant. You answer user questions based on provided context. If you can't answer the question with the provided context, say you don't know.<|eot_id|>
+
+<|start_header_id|>user<|end_header_id|>
+User Query:
+{query}
+
+Context:
+{context}<|eot_id|>
+
+<|start_header_id|>assistant<|end_header_id|>
+"""
+
+rag_prompt = PromptTemplate.from_template(RAG_PROMPT_TEMPLATE)
+
+# -- GENERATION -- #
+"""
+1. Create a HuggingFaceEndpoint for the LLM
+"""
+hf_llm = HuggingFaceEndpoint(
+    endpoint_url=HF_LLM_ENDPOINT,
+    max_new_tokens=512,
+    top_k=10,
+    top_p=0.95,
+    temperature=0.3,
+    repetition_penalty=1.15,
+    huggingfacehub_api_token=HF_TOKEN,
+)
+
+@cl.author_rename
+def rename(original_author: str):
+    """
+    This function can be used to rename the 'author' of a message. 
+
+    In this case, we're overriding the 'Assistant' author to be 'Paul Graham Essay Bot'.
+    """
+    rename_dict = {
+        "Assistant" : "Paul Graham Essay Bot"
+    }
+    return rename_dict.get(original_author, original_author)
+
+@cl.on_chat_start
+async def start_chat():
+    """
+    This function will be called at the start of every user session. 
+
+    We will build our LCEL RAG chain here, and store it in the user session. 
+
+    The user session is a dictionary that is unique to each user session, and is stored in the memory of the server.
+    """
+
+    lcel_rag_chain = (
+        {"context": itemgetter("query") | hf_retriever, "query": itemgetter("query")}
+        | rag_prompt | hf_llm
+    )
+
+    cl.user_session.set("lcel_rag_chain", lcel_rag_chain)
+
+@cl.on_message  
+async def main(message: cl.Message):
+    """
+    This function will be called every time a message is recieved from a session.
+
+    We will use the LCEL RAG chain to generate a response to the user query.
+
+    The LCEL RAG chain is stored in the user session, and is unique to each user session - this is why we can access it here.
+    """
+    lcel_rag_chain = cl.user_session.get("lcel_rag_chain")
+
+    msg = cl.Message(content="")
+
+    for chunk in await cl.make_async(lcel_rag_chain.stream)(
+        {"query": message.content},
+        config=RunnableConfig(callbacks=[cl.LangchainCallbackHandler()]),
+    ):
+        await msg.stream_token(chunk)
+
+    await msg.send()