Upload 2 files

app.py

@@ -12,6 +12,8 @@ import base64
 
 st.set_page_config(layout="wide")
 os.environ["OPENAI_API_KEY"] = "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"
+os.environ["LANGCHAIN_TRACING_V2"] = "true"
+os.environ["LANGCHAIN_API_KEY"] = "ls__aca2f2f97d2f4b9caef0ef75c3c33f9d"
 
 get_references = lambda relevant_docs: " ".join(
     [f"[{ref}]" for ref in sorted([ref.metadata["chunk_id"] for ref in relevant_docs])]
@@ -46,14 +48,18 @@ def process_documents_wrapper(inputs):
     snippets = process_documents(inputs)
     st.session_state.retriever = create_retriever(snippets)
     st.session_state.source_doc_urls = inputs
-    st.session_state.index = [snip.metadata["header"] for snip in snippets]
+    st.session_state.index = [
+        [snip.metadata["chunk_id"], snip.metadata["header"]] for snip in snippets
+    ]
     response = f"Uploaded and processed documents {inputs}"
     st.session_state.messages.append((f"/upload {inputs}", response, ""))
     return response
 
 
 def index_documents_wrapper(inputs=None):
-    response = pd.Series(st.session_state.index, name="references").to_markdown()
+    response = pd.DataFrame(
+        st.session_state.index, columns=["id", "reference"]
+    ).to_markdown()
     st.session_state.messages.append(("/index", response, ""))
     return response
 
@@ -173,4 +179,4 @@ if __name__ == "__main__":
         default_function=query_llm_wrapper,
         all_commands=all_commands,
     )
-    boot(command_center)
+    boot(command_center)

process_documents.py

@@ -10,17 +10,32 @@ deep_strip = lambda text: re.sub(r"\s+", " ", text or "").strip()
 
 def process_documents(urls):
     snippets = []
-    for url in urls:
+    for source_id, url in enumerate(urls):
         if url.endswith(".pdf"):
-            snippets.extend(process_pdf(url))
+            snippets.extend(process_pdf(url, source_id))
         else:
-            snippets.extend(process_web(url))
-    for e, snippet in enumerate(snippets):
-        snippet.metadata["chunk_id"] = e
+            snippets.extend(process_web(url, source_id))
     return snippets
 
 
-def process_pdf(url):
+def process_web(url, source_id):
+    data = WebBaseLoader(url).load()[0]
+    document_snippets = [
+        Document(
+            page_content=deep_strip(data.page_content),
+            metadata={
+                "header": data.metadata["title"],
+                "source_url": url,
+                "source_type": "web",
+                "chunk_id": f"{source_id}_0",
+                "source_id": source_id,
+            },
+        )
+    ]
+    return document_snippets
+
+
+def process_pdf(url, source_id):
     data = PDFMinerPDFasHTMLLoader(url).load()[0]
     content = BeautifulSoup(data.page_content, "html.parser").find_all("div")
     snippets = get_pdf_snippets(content)
@@ -36,7 +51,8 @@ def process_pdf(url):
                 "header": " ".join(snip[1]["header_text"].split()[:10]),
                 "source_url": url,
                 "source_type": "pdf",
-                "chunk_id": i,
+                "chunk_id": f"{source_id}_{i}",
+                "source_id": source_id,
             },
         )
         for i, snip in enumerate(semantic_snippets)
@@ -123,19 +139,3 @@ def get_pdf_semantic_snippets(filtered_snippets, median_font_size):
         }
         semantic_snippets.append((current_content, metadata))
     return semantic_snippets
-
-
-def process_web(url):
-    data = WebBaseLoader(url).load()[0]
-    document_snippets = [
-        Document(
-            page_content=deep_strip(data.page_content),
-            metadata={
-                "header": data.metadata["title"],
-                "source_url": url,
-                "source_type": "web",
-                "chunk_id": 0,
-            },
-        )
-    ]
-    return document_snippets