Split app.py into two scripts for a better structure.

app.py

@@ -1,61 +1,11 @@
-# AI assistant with a RAG system to query information from the CAMELS cosmological simulations using Langchain
+# AI assistant with a RAG system to query information from the CAMELS cosmological simulations using Langchain and deployed with Gradio
 # Author: Pablo Villanueva Domingo
 
-import gradio as gr
-from langchain import hub
-from langchain_chroma import Chroma
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.runnables import RunnablePassthrough
+from rag import RAG, load_docs
 from langchain_community.embeddings import HuggingFaceInstructEmbeddings
-from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_mistralai import ChatMistralAI
-from langchain_community.document_loaders import WebBaseLoader
 from langchain_core.rate_limiters import InMemoryRateLimiter
-
-# Load documentation from urls
-def get_docs():
-
-    # Get urls
-    urlsfile = open("urls.txt")
-    urls = urlsfile.readlines()
-    urls = [url.replace("\n","") for url in urls]
-    urlsfile.close()
-
-    # Load, chunk and index the contents of the blog.
-    loader = WebBaseLoader(urls)
-    docs = loader.load()
-
-    return docs
-
-# Join content pages for processing
-def format_docs(docs):
-    return "\n\n".join(doc.page_content for doc in docs)
-
-# Create a RAG chain
-def RAG(llm, docs, embeddings):
-
-    # Split text
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
-    splits = text_splitter.split_documents(docs)
-
-    # Create vector store
-    vectorstore = Chroma.from_documents(documents=splits, embedding=embeddings)
-
-    # Retrieve and generate using the relevant snippets of the documents
-    retriever = vectorstore.as_retriever()
-
-    # Prompt basis example for RAG systems
-    prompt = hub.pull("rlm/rag-prompt")
-
-    # Create the chain
-    rag_chain = (
-        {"context": retriever | format_docs, "question": RunnablePassthrough()}
-        | prompt
-        | llm
-        | StrOutputParser()
-    )
-
-    return rag_chain
+import gradio as gr
 
 # Define a limiter to avoid rate limit issues with MistralAI
 rate_limiter = InMemoryRateLimiter(
@@ -64,8 +14,8 @@ rate_limiter = InMemoryRateLimiter(
     max_bucket_size=10,  # Controls the maximum burst size.
 )
 
-# Get docs
-docs = get_docs()
+# Load the documentation
+docs = load_docs()
 print("Pages loaded:",len(docs))
 
 # LLM model
@@ -117,7 +67,7 @@ if __name__=="__main__":
                             examples=example_questions,
                             theme=gr.themes.Soft(),
                             description=description,
-                            cache_examples=False,
+                            #cache_examples=False,
                             chatbot=chatbot)
 
     demo.launch()

rag.py

@@ -0,0 +1,54 @@
+# Utilities to build a RAG system to query information from the CAMELS cosmological simulations using Langchain
+# Author: Pablo Villanueva Domingo
+
+from langchain import hub
+from langchain_chroma import Chroma
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnablePassthrough
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import WebBaseLoader
+
+# Load documentation from urls
+def load_docs():
+
+    # Get urls
+    urlsfile = open("urls.txt")
+    urls = urlsfile.readlines()
+    urls = [url.replace("\n","") for url in urls]
+    urlsfile.close()
+
+    # Load, chunk and index the contents of the blog.
+    loader = WebBaseLoader(urls)
+    docs = loader.load()
+
+    return docs
+
+# Join content pages for processing
+def format_docs(docs):
+    return "\n\n".join(doc.page_content for doc in docs)
+
+# Create a RAG chain
+def RAG(llm, docs, embeddings):
+
+    # Split text
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+    splits = text_splitter.split_documents(docs)
+
+    # Create vector store
+    vectorstore = Chroma.from_documents(documents=splits, embedding=embeddings)
+
+    # Retrieve and generate using the relevant snippets of the documents
+    retriever = vectorstore.as_retriever()
+
+    # Prompt basis example for RAG systems
+    prompt = hub.pull("rlm/rag-prompt")
+
+    # Create the chain
+    rag_chain = (
+        {"context": retriever | format_docs, "question": RunnablePassthrough()}
+        | prompt
+        | llm
+        | StrOutputParser()
+    )
+
+    return rag_chain