added Hybrid Search with BM25

app.py

@@ -28,6 +28,7 @@ from langchain_community.llms.huggingface_pipeline import HuggingFacePipeline
 from langchain_core.indexing import index
 from langchain_core.vectorstores import VectorStore
 from llama_index.core.node_parser import TextSplitter
+from langchain.retrievers import BM25Retriever, EnsembleRetriever
 from llama_index.legacy.vector_stores import FaissVectorStore
 from pycparser.ply.yacc import token
 from ragatouille import RAGPretrainedModel
@@ -90,6 +91,10 @@ class BSIChatbot:
         global vectorstore
         RAW_KNOWLEDGE_BASE = []
 
+        #Qdrant:
+        #client = QdrantClient(path=saved_db_path)
+        #db = Qdrant(client=client, collection_name=self.collection_name, embeddings=embeddings, )
+
         #Embedding, Vector generation and storing:
         self.embedding_model = HuggingFaceEmbeddings(
             model_name=self.word_and_embed_model_path,
@@ -259,6 +264,19 @@ class BSIChatbot:
         print(f"printing first chunk to see whats inside: {retrieved_chunks[0]}")
         return retrieved_chunks
 
+    def retrieveDocFromFaiss(self):
+        global vectorstore
+
+        all_documents = []
+
+        # Iteriere über alle IDs im index_to_docstore_id
+        for doc_id in vector_store.index_to_docstore_id.values():
+            # Hole das Dokument aus dem docstore
+            document = vector_store.docstore[doc_id]
+            all_documents.append(document)
+
+        return all_documents
+
     def initializeLLM(self):
         bnb_config = BitsAndBytesConfig(
             load_in_8bit=True,
@@ -292,8 +310,19 @@ class BSIChatbot:
         rerankingModel = RAGPretrainedModel.from_pretrained(self.rerankModelPath)
 
 
-    def retrieval(self, query, rerankingStep):
-        retrieved_chunks = self.retrieveSimiliarEmbedding(query)
+    def retrieval(self, query, rerankingStep, hybridSearch):
+        global vectorstore
+        if hybridSearch == True:
+            allDocs = self.retrieveDocFromFaiss()
+            bm25_retriever = BM25Retriever.from_documents(allDocs.page_content)
+            #TODO!
+            bm25_retriever.k= 4
+            vectordb = vectorstore.as_retriever(search_kwargs={"k":4})
+            ensemble_retriever = EnsembleRetriever(retrievers=[bm25_retriever, vectordb], weights=[0.5, 0.5])
+            retrieved_chunks = ensemble_retriever.get_relevant_documents(query)
+            print("DBG: Number of Chunks retrieved" +len(retrieved_chunks))
+        else:
+            retrieved_chunks = self.retrieveSimiliarEmbedding(query)
         retrieved_chunks_text = []
         # TODO Irgendwas stimmt hier mit den Listen nicht
         for chunk in retrieved_chunks:
@@ -315,7 +344,7 @@ class BSIChatbot:
                 self.initializeRerankingModel()
             print("Starting Reranking Chunks...")
             rerankingModel
-            retrieved_chunks_text = rerankingModel.rerank(query, retrieved_chunks_text, k=5)
+            retrieved_chunks_text = rerankingModel.rerank(query, retrieved_chunks_text, k=15)
             retrieved_chunks_text = [chunk["content"] for chunk in retrieved_chunks_text]
 
             i = 1