Update app.py

app.py

@@ -17,11 +17,17 @@ from langchain.document_loaders import PyPDFLoader
 # Use Chroma in Colab to create vector embeddings, I then saved them to HuggingFace so now I have to set it use them here.
 #from chromadb.config import Settings
 #client = chromadb.Client(Settings(
-#    chroma_db_impl="duckdb+parquet",
+##    chroma_db_impl="duckdb+parquet",
 #    persist_directory="./embeddings" # Optional, defaults to .chromadb/ in the current directory
 #))
 
 
+
+
+
+
+
+
 def get_empty_state():
     return {"total_tokens": 0, "messages": []}
 
@@ -60,6 +66,27 @@ def on_prompt_template_change(prompt_template):
 def submit_message(prompt, prompt_template, temperature, max_tokens, context_length, state):
 
     openai.api_key = os.environ['openai_key']
+
+    
+    # load in all the files
+    path = './files'
+    #pdf_files = glob.glob(os.path.join(path, "*.pdf"))
+    pdf_files = glob.glob(os.path.join(path, "*.pdf"))
+
+    for file in pdf_files:
+      loader = PyPDFLoader(file)
+      pages = loader.load_and_split()
+      text_splitter = TokenTextSplitter(chunk_size=1000, chunk_overlap=0)
+      split_pages = text_splitter.split_documents(pages)
+     
+    persist_directory = "./embeddings"
+    embeddings = OpenAIEmbeddings()
+    vectordb = Chroma.from_documents(split_pages, embeddings, persist_directory=persist_directory)
+    vectordb.persist()
+
+
+
+    
     history = state['messages']
 
     if not prompt:
@@ -79,11 +106,9 @@ def submit_message(prompt, prompt_template, temperature, max_tokens, context_len
 
 # completion = openai.ChatCompletion.create(model="gpt-3.5-turbo", messages=system_prompt + history[-context_length*2:] + [prompt_msg], temperature=temperature, max_tokens=max_tokens)
 
-        persist_directory = "./embeddings"
-        vectordb = Chroma.from_documents(romeoandjuliet_doc, embeddings, persist_directory=persist_directory)
         completion = ChatVectorDBChain.from_llm(OpenAI(temperature=temperature, max_tokens=max_tokens, model_name="gpt-3.5-turbo"), vectordb, return_source_documents=True)
         result = completion({"question": system_prompt + history[-context_length*2:] + [prompt_msg]})
-# from https://blog.devgenius.io/chat-with-document-s-using-openai-chatgpt-api-and-text-embedding-6a0ce3dc8bc8
+        # from https://blog.devgenius.io/chat-with-document-s-using-openai-chatgpt-api-and-text-embedding-6a0ce3dc8bc8
         
         history.append(prompt_msg)
         history.append(completion.choices[0].message.to_dict())