Update process_documents.py

process_documents.py

@@ -36,14 +36,16 @@ def process_documents(urls):
 def process_web(url, source_id):
     data = WebBaseLoader(f"https://r.jina.ai/{url}").load()[0]
     try:
+        header = re.search(r"Title: (.*)?", data.page_content).group(1)
         page_content = data.page_content[data.page_content.index("Markdown Content:") + len("Markdown Content:"):].strip()
     except Exception as e:
+        header = ""
         page_content = data.page_content.strip()
     document_snippets = [
         Document(
             page_content=page_content,
             metadata={
-                "header": data.metadata["title"],
+                "header": header,
                 "source_url": url,
                 "source_type": "web",
                 "chunk_id": source_id,