Spaces:
Runtime error
Runtime error
André Oriani
commited on
Commit
·
43b89fd
1
Parent(s):
20b588a
I wasn't properly using the embeddings cache
Browse files
app.py
CHANGED
@@ -12,6 +12,7 @@ from langchain_core.runnables.passthrough import RunnablePassthrough
|
|
12 |
from langchain_core.output_parsers import StrOutputParser
|
13 |
from langchain_core.prompts import ChatPromptTemplate
|
14 |
from langchain_openai import ChatOpenAI
|
|
|
15 |
|
16 |
print("""
|
17 |
=================================================================================
|
@@ -35,13 +36,13 @@ chunked_documents = text_splitter.split_documents(data)
|
|
35 |
# Store the chunked documents in the vector store if that was not done already
|
36 |
embedding_model = OpenAIEmbeddings()
|
37 |
store = LocalFileStore("./cache/")
|
38 |
-
|
39 |
index_path = "faiss_index"
|
40 |
if os.path.exists(index_path):
|
41 |
-
vector_store = FAISS.load_local(index_path,
|
42 |
print("Vector store loaded from saved index.")
|
43 |
else:
|
44 |
-
vector_store = FAISS.from_documents(chunked_documents,
|
45 |
print("Vector store created from documents.")
|
46 |
vector_store.save_local(index_path)
|
47 |
print("Vector store saved locally.")
|
|
|
12 |
from langchain_core.output_parsers import StrOutputParser
|
13 |
from langchain_core.prompts import ChatPromptTemplate
|
14 |
from langchain_openai import ChatOpenAI
|
15 |
+
import asyncio
|
16 |
|
17 |
print("""
|
18 |
=================================================================================
|
|
|
36 |
# Store the chunked documents in the vector store if that was not done already
|
37 |
embedding_model = OpenAIEmbeddings()
|
38 |
store = LocalFileStore("./cache/")
|
39 |
+
cached_embedder = CacheBackedEmbeddings.from_bytes_store(embedding_model, store, namespace=embedding_model.model)
|
40 |
index_path = "faiss_index"
|
41 |
if os.path.exists(index_path):
|
42 |
+
vector_store = FAISS.load_local(index_path, cached_embedder, allow_dangerous_deserialization=True)
|
43 |
print("Vector store loaded from saved index.")
|
44 |
else:
|
45 |
+
vector_store = FAISS.from_documents(chunked_documents, cached_embedder)
|
46 |
print("Vector store created from documents.")
|
47 |
vector_store.save_local(index_path)
|
48 |
print("Vector store saved locally.")
|