André Oriani commited on
Commit
43b89fd
·
1 Parent(s): 20b588a

I wasn't properly using the embeddings cache

Browse files
Files changed (1) hide show
  1. app.py +4 -3
app.py CHANGED
@@ -12,6 +12,7 @@ from langchain_core.runnables.passthrough import RunnablePassthrough
12
  from langchain_core.output_parsers import StrOutputParser
13
  from langchain_core.prompts import ChatPromptTemplate
14
  from langchain_openai import ChatOpenAI
 
15
 
16
  print("""
17
  =================================================================================
@@ -35,13 +36,13 @@ chunked_documents = text_splitter.split_documents(data)
35
  # Store the chunked documents in the vector store if that was not done already
36
  embedding_model = OpenAIEmbeddings()
37
  store = LocalFileStore("./cache/")
38
- embedder = CacheBackedEmbeddings.from_bytes_store(embedding_model, store, namespace=embedding_model.model)
39
  index_path = "faiss_index"
40
  if os.path.exists(index_path):
41
- vector_store = FAISS.load_local(index_path, embedding_model, allow_dangerous_deserialization=True)
42
  print("Vector store loaded from saved index.")
43
  else:
44
- vector_store = FAISS.from_documents(chunked_documents, embedding_model)
45
  print("Vector store created from documents.")
46
  vector_store.save_local(index_path)
47
  print("Vector store saved locally.")
 
12
  from langchain_core.output_parsers import StrOutputParser
13
  from langchain_core.prompts import ChatPromptTemplate
14
  from langchain_openai import ChatOpenAI
15
+ import asyncio
16
 
17
  print("""
18
  =================================================================================
 
36
  # Store the chunked documents in the vector store if that was not done already
37
  embedding_model = OpenAIEmbeddings()
38
  store = LocalFileStore("./cache/")
39
+ cached_embedder = CacheBackedEmbeddings.from_bytes_store(embedding_model, store, namespace=embedding_model.model)
40
  index_path = "faiss_index"
41
  if os.path.exists(index_path):
42
+ vector_store = FAISS.load_local(index_path, cached_embedder, allow_dangerous_deserialization=True)
43
  print("Vector store loaded from saved index.")
44
  else:
45
+ vector_store = FAISS.from_documents(chunked_documents, cached_embedder)
46
  print("Vector store created from documents.")
47
  vector_store.save_local(index_path)
48
  print("Vector store saved locally.")