Update utils.py

utils.py

@@ -3,7 +3,7 @@ from langchain.embeddings import OpenAIEmbeddings
 from langchain.vectorstores import Chroma
 
 
-# loading PDF, DOCX and TXT files as LangChain Documents
+# stack up loading methods using elif statments for loading PDF, DOCX, TXT, and CSV files into LangChain Documents
 def load_document(file):
     import os
     name, extension = os.path.splitext(file)
@@ -30,7 +30,7 @@ def load_document(file):
     return data
 
 
-# splitting data in chunks
+# chunck your data for embedding
 def chunk_data(data, chunk_size=256, chunk_overlap=20):
     from langchain.text_splitter import RecursiveCharacterTextSplitter
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
@@ -38,33 +38,27 @@ def chunk_data(data, chunk_size=256, chunk_overlap=20):
     return chunks
 
 
-# create embeddings using OpenAIEmbeddings() and save them in a Chroma vector store
+# using OpenAIEmbeddings() create your embeddings and save to the Chroma vector store
 def create_embeddings(chunks):
     embeddings = OpenAIEmbeddings()
     vector_store = Chroma.from_documents(chunks, embeddings)
-
-    # if you want to use a specific directory for chromadb
-    # vector_store = Chroma.from_documents(chunks, embeddings, persist_directory='./mychroma_db')
     return vector_store
 
-
+# here where you ask your question, here we use a combination of RetrievalQA and ChatOpenAI but his is not the only way to do this
 def ask_and_get_answer(vector_store, q, k=3):
     from langchain.chains import RetrievalQA
     from langchain.chat_models import ChatOpenAI
-
+    # choose the 3.5 turbo model which is default and set the temperature to 1 which is maximum
     llm = ChatOpenAI(model='gpt-3.5-turbo', temperature=1)
     retriever = vector_store.as_retriever(search_type='similarity', search_kwargs={'k': k})
     chain = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever)
-
     answer = chain.run(q)
     return answer
 
 
-# calculate embedding cost using tiktoken
+# return the embedding cost (using tiktoken)
 def calculate_embedding_cost(texts):
     import tiktoken
     enc = tiktoken.encoding_for_model('text-embedding-ada-002')
     total_tokens = sum([len(enc.encode(page.page_content)) for page in texts])
-    # print(f'Total Tokens: {total_tokens}')
-    # print(f'Embedding Cost in USD: {total_tokens / 1000 * 0.0004:.6f}')
     return total_tokens, total_tokens / 1000 * 0.0004