added EvalDataset Generation

app.py

@@ -56,6 +56,7 @@ from huggingface_hub import login
 login(token=HF_KEY)
 vectorstore=None
 rerankingModel=None
+bm25_retriever=None
 
 class BSIChatbot:
     embedding_model = None
@@ -158,14 +159,14 @@ class BSIChatbot:
             ##Was macht man mit start Index herausfinden und wie metadata adden
             docs_processed = []
             for doc in RAW_KNOWLEDGE_BASE:
-                print(f"Word-Length in doc:{len(doc.page_content.split())}")
+                #newprint(f"Word-Length in doc:{len(doc.page_content.split())}")
                 doc_cache = markdown_splitter.split_text(doc.page_content)
                 # print(f"Word-Length in doc_cache after MarkdownSplitter:{len(doc_cache.split())}")
                 doc_cache = text_splitter.split_documents(doc_cache)
                 # print(f"Word-Length in doc_cache after text_splitter:{len(doc_cache.split())}")
                 for chunk in doc_cache:
                     chunk.metadata.update({"source": doc.metadata['source']})
-                    print(f"Chunk_Debug len: {len(chunk.page_content.split())} and Chunk:{chunk}")
+                    #newprint(f"Chunk_Debug len: {len(chunk.page_content.split())} and Chunk:{chunk}")
                 # DEBUG:
                 # print(f"doc_cache after Metadata added:{doc_cache}\n")
                 docs_processed += doc_cache
@@ -197,7 +198,7 @@ class BSIChatbot:
             #for doc in RAW_KNOWLEDGE_BASE:
             #    docs_processed += text_splitter.split_documents([doc])
 
-            print(f"Docs processed:{len(docs_processed)}")
+            #newprint(f"Docs processed:{len(docs_processed)}")
             # Max_Sequence_Length of e5 large instr = 512 Tokens
 
 
@@ -230,14 +231,14 @@ class BSIChatbot:
             vectorstore.save_local(self.embedPath)
             #self.vectorstore.index = index_gpu
             end = time.time()
-            print("Saving Embeddings took", end-start, "seconds!")
+            #newprint("Saving Embeddings took", end-start, "seconds!")
         else:
             start = time.time()
             vectorstore = FAISS.load_local(self.embedPath, self.embedding_model, allow_dangerous_deserialization=True)
             #self.vectorstore.index = index_gpu
             end = time.time()
 
-            print("Loading Embeddings took", end - start, "seconds!")
+            #newprint("Loading Embeddings took", end - start, "seconds!")
 
     def retrieveSimiliarEmbedding(self, query):
         global vectorstore
@@ -254,7 +255,7 @@ class BSIChatbot:
         #        finalchunks.append(chunk)
         #retrieved_chunks = finalchunks
         end = time.time()
-        print("Retrieving Chunks with similiar embeddings took", end - start, "seconds!")
+        #newrint("Retrieving Chunks with similiar embeddings took", end - start, "seconds!")
         #print("\n==================================Top document==================================")
         #print(retrieved_chunks[0].page_content)
         #print(retrieved_chunks[1].page_content)
@@ -263,7 +264,7 @@ class BSIChatbot:
         #print(retrieved_chunks[0].metadata)
         #print(retrieved_chunks[1].metadata)
         #print(retrieved_chunks[2].metadata)
-        print(f"printing first chunk to see whats inside: {retrieved_chunks[0]}")
+        #newprint(f"printing first chunk to see whats inside: {retrieved_chunks[0]}")
         return retrieved_chunks
 
     def retrieveDocFromFaiss(self):
@@ -325,17 +326,19 @@ class BSIChatbot:
 
     def retrieval(self, query, rerankingStep, hybridSearch):
         global vectorstore
+        global bm25_retriever
         if hybridSearch == True:
             allDocs = self.retrieveDocFromFaiss()
-            bm25_retriever = BM25Retriever.from_documents(allDocs)
+            if bm25_retriever == None:
+                bm25_retriever = BM25Retriever.from_documents(allDocs)
             #TODO!
             retriever_k=15
             bm25_retriever.k= retriever_k
             vectordb = vectorstore.as_retriever(search_kwargs={"k":retriever_k})
             ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, vectordb], weights=[0.5, 0.5])
             retrieved_chunks = ensemble_retriever.get_relevant_documents(query)
-            print("DBG: Number of Chunks retrieved")
-            print(len(retrieved_chunks))
+            #newprint("DBG: Number of Chunks retrieved")
+            #newprint(len(retrieved_chunks))
         else:
             retrieved_chunks = self.retrieveSimiliarEmbedding(query)
         retrieved_chunks_text = []
@@ -442,6 +445,7 @@ class BSIChatbot:
 
         # Alles außer letzte Useranfrage, Normaler Query
         query, context = self.retrieval(query, rerankingStep, True)
+
         if stepBackPrompt == True:
             stepBackQuery = self.stepBackPrompt(query)
             print("DBG stepBackQuery:" + stepBackQuery)
@@ -474,7 +478,7 @@ class BSIChatbot:
         #    question=query, context=context, history=history[:-1]
         # )
 
-        print(f"Query:\n{final_prompt}")
+        #newprint(f"Query:\n{final_prompt}")
         pattern = r"Filename:(.*?);"
         last_value = final_prompt[-1]["content"]