preload vector store

.gitignore

@@ -0,0 +1,49 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual Environment
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.conda/
+
+# VS Code
+.vscode/
+*.code-workspace
+.history/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# Local development files
+*.log
+.DS_Store
+Thumbs.db
+
+# Project specific
+.env
+*.db
+*.sqlite3

agent.py

@@ -1,25 +1,30 @@
 """LangGraph Agent"""
+
 import os
+
 from dotenv import load_dotenv
-from langgraph.graph import START, StateGraph, MessagesState
-from langgraph.prebuilt import tools_condition
-from langgraph.prebuilt import ToolNode
-from langchain_google_genai import ChatGoogleGenerativeAI
-from langchain_openai import ChatOpenAI
-from langchain.agents import initialize_agent, Tool
-from langchain_groq import ChatGroq
-from langchain_huggingface import ChatHuggingFace, HuggingFaceEndpoint, HuggingFaceEmbeddings
+from langchain.agents import Tool, initialize_agent
+from langchain.tools.retriever import create_retriever_tool
+from langchain_community.document_loaders import ArxivLoader, WikipediaLoader
 from langchain_community.tools.tavily_search import TavilySearchResults
-from langchain_community.document_loaders import WikipediaLoader
-from langchain_community.document_loaders import ArxivLoader
 from langchain_community.vectorstores import SupabaseVectorStore
-from langchain_core.messages import SystemMessage, HumanMessage
+from langchain_core.messages import HumanMessage, SystemMessage
 from langchain_core.tools import tool
-from langchain.tools.retriever import create_retriever_tool
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_groq import ChatGroq
+from langchain_huggingface import (
+    ChatHuggingFace,
+    HuggingFaceEmbeddings,
+    HuggingFaceEndpoint,
+)
+from langchain_openai import ChatOpenAI
+from langgraph.graph import START, MessagesState, StateGraph
+from langgraph.prebuilt import ToolNode, tools_condition
 from supabase.client import Client, create_client
 
 load_dotenv()
 
+
 @tool
 def multiply(a: int, b: int) -> int:
     """Multiply two numbers.
@@ -29,6 +34,7 @@ def multiply(a: int, b: int) -> int:
     """
     return a * b
 
+
 @tool
 def add(a: int, b: int) -> int:
     """Add two numbers.
@@ -38,6 +44,7 @@ def add(a: int, b: int) -> int:
     """
     return a + b
 
+
 @tool
 def subtract(a: int, b: int) -> int:
     """Subtract two numbers.
@@ -47,6 +54,7 @@ def subtract(a: int, b: int) -> int:
     """
     return a - b
 
+
 @tool
 def divide(a: int, b: int) -> int:
     """Divide two numbers.
@@ -58,6 +66,7 @@ def divide(a: int, b: int) -> int:
         raise ValueError("Cannot divide by zero.")
     return a / b
 
+
 @tool
 def modulus(a: int, b: int) -> int:
     """Get the modulus of two numbers.
@@ -67,6 +76,7 @@ def modulus(a: int, b: int) -> int:
     """
     return a % b
 
+
 @tool
 def wiki_search(query: str) -> str:
     """Search Wikipedia for a query and return maximum 2 results.
@@ -77,17 +87,22 @@ def wiki_search(query: str) -> str:
         [
             f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
             for doc in search_docs
-        ])
+        ]
+    )
     return {"wiki_results": formatted_search_docs}
 
+
 @tool
 def web_search(query: str) -> str:
     """Search Tavily for a query and return maximum 3 results.
     Args:
         query: The search query."""
-    search_docs = TavilySearchResults(max_results=3).invoke(query)  # Fixed: pass query as positional argument
+    search_docs = TavilySearchResults(max_results=3).invoke(
+        query
+    )  # Fixed: pass query as positional argument
     return {"web_results": search_docs}  # Also fixed the return type issue
 
+
 @tool
 def arvix_search(query: str) -> str:
     """Search Arxiv for a query and return maximum 3 result.
@@ -98,24 +113,24 @@ def arvix_search(query: str) -> str:
         [
             f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content[:1000]}\n</Document>'
             for doc in search_docs
-        ])
+        ]
+    )
     return {"arvix_results": formatted_search_docs}
 
 
 def test_supabase_connection():
     load_dotenv()
-    
+
     try:
         supabase = create_client(
-            os.environ.get("SUPABASE_URL"),
-            os.environ.get("SUPABASE_SERVICE_KEY")
+            os.environ.get("SUPABASE_URL"), os.environ.get("SUPABASE_SERVICE_KEY")
         )
-        
+
         # Test query
-        result = supabase.table('documents').select("*").limit(1).execute()
+        result = supabase.table("documents").select("*").limit(1).execute()
         print("Connection successful!")
         return True
-        
+
     except Exception as e:
         print(f"Connection failed: {e}")
         return False
@@ -129,13 +144,15 @@ with open("system_prompt.txt", "r", encoding="utf-8") as f:
 sys_msg = SystemMessage(content=system_prompt)
 
 # build a retriever
-embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2") #  dim=768
+embeddings = HuggingFaceEmbeddings(
+    model_name="sentence-transformers/all-mpnet-base-v2"
+)  #  dim=768
 supabase: Client = create_client(
-    os.environ.get("SUPABASE_URL"),
-    os.environ.get("SUPABASE_SERVICE_KEY"))
+    os.environ.get("SUPABASE_URL"), os.environ.get("SUPABASE_SERVICE_KEY")
+)
 vector_store = SupabaseVectorStore(
     client=supabase,
-    embedding= embeddings,
+    embedding=embeddings,
     table_name="documents",
     query_name="match_documents_langchain",
 )
@@ -158,16 +175,21 @@ tools = [
     arvix_search,
 ]
 
+
 # Build graph function
-def build_graph(provider: str = "openai"):
+def build_graph(provider: str = "google"):
     """Build the graph"""
     # Load environment variables from .env file
     if provider == "google":
         # Google Gemini
-        llm = ChatGoogleGenerativeAI(model="gemini-2.5-flash-preview-05-20", temperature=0)
+        llm = ChatGoogleGenerativeAI(
+            model="gemini-2.5-flash-preview-05-20", temperature=0
+        )
     elif provider == "groq":
         # Groq https://console.groq.com/docs/models
-        llm = ChatGroq(model="qwen-qwq-32b", temperature=0) # optional : qwen-qwq-32b gemma2-9b-it
+        llm = ChatGroq(
+            model="qwen-qwq-32b", temperature=0
+        )  # optional : qwen-qwq-32b gemma2-9b-it
     elif provider == "openai":
         # OpenAI
         llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
@@ -192,7 +214,9 @@ def build_graph(provider: str = "openai"):
         """Retriever node"""
         try:
             # Use the vector store to find similar questions
-            similar_question = vector_store.similarity_search(state["messages"][0].content)
+            similar_question = vector_store.similarity_search(
+                state["messages"][0].content
+            )
             if not similar_question:
                 raise ValueError("No similar questions found.")
         except Exception as e:

util.py

@@ -0,0 +1,56 @@
+import ast
+import logging
+import os
+
+import pandas as pd
+from dotenv import load_dotenv
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import SupabaseVectorStore
+from supabase.client import create_client
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class SupabaseConnector:
+    def __init__(self):
+        load_dotenv()
+        self.supabase = create_client(
+            os.environ.get("SUPABASE_URL"), os.environ.get("SUPABASE_SERVICE_KEY")
+        )
+        self.embeddings = HuggingFaceEmbeddings(
+            model_name="sentence-transformers/all-mpnet-base-v2"
+        )
+        self.vector_store = SupabaseVectorStore(
+            client=self.supabase,
+            embedding=self.embeddings,
+            table_name="documents",
+            query_name="match_documents_langchain",
+        )
+
+    def upload_csv(self, file_path: str, batch_size: int = 100):
+        """
+        Upload documents from supabase_docs.csv to Supabase vector store.
+        Only 'content' and parsed 'metadata' are used.
+        """
+        df = pd.read_csv(file_path)
+        logger.info(f"Loaded {len(df)} records from {file_path}")
+
+        # Parse metadata column from string to dict
+        df["metadata"] = df["metadata"].apply(
+            lambda x: ast.literal_eval(x) if isinstance(x, str) else {}
+        )
+
+        for i in range(0, len(df), batch_size):
+            batch = df.iloc[i : i + batch_size]
+            texts = batch["content"].tolist()
+            metadatas = batch["metadata"].tolist()
+            self.vector_store.add_texts(texts=texts, metadatas=metadatas)
+            logger.info(f"Uploaded batch {i//batch_size + 1}")
+
+        logger.info("CSV upload completed.")
+
+
+if __name__ == "__main__":
+    connector = SupabaseConnector()
+    connector.upload_csv("supabase_docs.csv")