try Chromadb version

.gitignore

@@ -0,0 +1,2 @@
+
+.env

app.py

@@ -1,20 +1,48 @@
 import chainlit as cl
 from datasets import load_dataset
 from langchain_community.document_loaders import CSVLoader
+from langchain_community.vectorstores.chroma import Chroma
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_openai import OpenAIEmbeddings
-from langchain.embeddings import CacheBackedEmbeddings
-from langchain.storage import LocalFileStore
-from langchain_community.vectorstores import FAISS
-from langchain_core.runnables.base import RunnableSequence
+#from langchain.embeddings import CacheBackedEmbeddings
+#from langchain.storage import LocalFileStore
+#from langchain_community.vectorstores import FAISS
+#from langchain_core.runnables.base import RunnableSequence
 from langchain_core.runnables.passthrough import RunnablePassthrough
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.prompts import ChatPromptTemplate
 from langchain_openai import ChatOpenAI
 from langchain.schema.runnable import Runnable, RunnablePassthrough, RunnableConfig
 from langchain.callbacks.base import BaseCallbackHandler
+from langchain.indexes import SQLRecordManager, index
+
+# def setup_data():
+#     dataset = load_dataset("ShubhamChoksi/IMDB_Movies")
+#     dataset_dict = dataset
+#     dataset_dict["train"].to_csv("imdb.csv")
+
+#     loader = CSVLoader(file_path="imdb.csv")
+#     data = loader.load()
+
+#     text_splitter = RecursiveCharacterTextSplitter(
+#         chunk_size=1000,
+#         chunk_overlap=100
+#     )
+
+#     chunked_documents = text_splitter.split_documents(data)
+#     embedding_model = OpenAIEmbeddings()
+
+#     store = LocalFileStore("./cache/")
+#     embedder = CacheBackedEmbeddings.from_bytes_store(embedding_model, store, namespace=embedding_model.model)
+
+#     vector_store = FAISS.from_documents(chunked_documents, embedder)
+#     vector_store.save_local("faiss_index")
+
+#     return vector_store
+
 
 def setup_data():
+
     dataset = load_dataset("ShubhamChoksi/IMDB_Movies")
     dataset_dict = dataset
     dataset_dict["train"].to_csv("imdb.csv")
@@ -27,17 +55,28 @@ def setup_data():
         chunk_overlap=100
     )
 
-    chunked_documents = text_splitter.split_documents(data)
-    embedding_model = OpenAIEmbeddings()
+    docs = text_splitter.split_documents(data) # chunked documents
+    embeddings_model = OpenAIEmbeddings()
 
-    store = LocalFileStore("./cache/")
-    embedder = CacheBackedEmbeddings.from_bytes_store(embedding_model, store, namespace=embedding_model.model)
+    doc_search = Chroma.from_documents(docs, embeddings_model)
 
-    vector_store = FAISS.from_documents(chunked_documents, embedder)
-    vector_store.save_local("faiss_index")
+    namespace = "chromadb/my_documents"
+    record_manager = SQLRecordManager(
+        namespace, db_url="sqlite:///record_manager_cache.sql"
+    )
+    record_manager.create_schema()
+
+    index_result = index(
+        docs,
+        record_manager,
+        doc_search,
+        cleanup="incremental",
+        source_id_key="source",
+    )
 
-    return vector_store
+    print(f"Indexing stats: {index_result}")
 
+    return doc_search
 
 doc_search = setup_data()
 model = ChatOpenAI(model_name="gpt-4o", temperature=0, streaming=True)

requirements.txt

@@ -5,4 +5,4 @@ langchain_openai
 faiss-cpu
 tiktoken
 chainlit
-
+chromadb