Upload 4 files

app.py

@@ -1,6 +1,6 @@
 import streamlit as st
 from generator import generate_response_from_document
-from retrieval import retrieve_documents_hybrid
+from retrieval import retrieve_documents_hybrid,find_query_dataset
 from evaluation import calculate_metrics
 from data_processing import load_recent_questions, save_recent_question
 import time
@@ -63,6 +63,8 @@ if "time_taken_for_response" not in st.session_state:
     st.session_state.time_taken_for_response = "N/A"
 if "metrics" not in st.session_state:
     st.session_state.metrics = {}
+if "metrics" not in st.session_state:
+    st.session_state.metrics = {}
 
 recent_data = load_recent_questions()
 
@@ -106,7 +108,8 @@ for q in reversed(recent_data["questions"]):  # Show latest first
 
 if st.button("Submit"):
     start_time = time.time()
-    st.session_state.retrieved_documents = retrieve_documents_hybrid(question, 10)  
+    st.session_state.query_dataset =  find_query_dataset(question)
+    st.session_state.retrieved_documents = retrieve_documents_hybrid(question, st.session_state.query_dataset, 10)  
     st.session_state.response = generate_response_from_document(question, st.session_state.retrieved_documents)
     end_time = time.time()
     st.session_state.time_taken_for_response = end_time - start_time
@@ -133,7 +136,7 @@ col1, col2 = st.columns([1, 3])  # Creating two columns for button and metrics d
 # Calculate Metrics Button
 with col1:
     if st.button("Show Metrics"):
-        st.session_state.metrics = calculate_metrics(question, st.session_state.response, st.session_state.retrieved_documents, st.session_state.time_taken_for_response)
+        st.session_state.metrics = calculate_metrics(question, st.session_state.query_dataset, st.session_state.response, st.session_state.retrieved_documents, st.session_state.time_taken_for_response)
     else:
         metrics_ = {}
 

evaluation.py

@@ -91,9 +91,8 @@ def adherence(response, relevant_documents):
 def compute_rmse(predicted_values, ground_truth_values):
     return np.sqrt(mean_squared_error(ground_truth_values, predicted_values))
 
-def calculate_metrics(question, response, docs, time_taken):
-    from retrieval import query_dataset
-    data = load_query_dataset(query_dataset)
+def calculate_metrics(question, q_dataset, response, docs, time_taken):
+    data = load_query_dataset(q_dataset)
     ground_truth_answer = retrieve_ground_truths(question, data)  # Store the ground truth answer
 
     # Ensure ground_truth_answer is not empty before proceeding

generator.py

@@ -11,12 +11,23 @@ def generate_response_from_document(query, retrieved_docs):
 
     # context = " ".join([doc.page_content for doc in retrieved_docs]) # Now iterates over Document objects
     context = " ".join([doc for doc in retrieved_docs])
-    prompt = (
-            "You are a highly intelligent assistant tasked with answering a question based strictly on the provided context. "
-            f"Given Question: {query} \n\n"
-            f"Context: {context} \n"
-            "Answer the question directly and concisely using only the information available in the context."
-        )
+    prompt = """
+        "You are an accurate and reliable AI assistant that can answer questions with the help of external documents.
+         Please note that external documents may contain noisy or factually incorrect information. 
+         If the information in the document contains the correct answer, you will give an accurate answer. 
+         If the information in the document does not contain the answer, you will generate ’I can not answer the question because of the insufficient information in documents.‘. 
+         If there are inconsistencies with the facts in some of the documents, please generate the response 'There are factual errors in the provided documents.' and provide the correct answer."
+
+         Context or Document: {context}
+         Query: {query}
+    """
+    # prompt = (
+    #         "You are a highly intelligent assistant tasked with answering a question based strictly on the provided context. "
+    #         f"Given Question: {query} \n\n"
+    #         f"Context: {context} \n"
+    #         f"Answer the question directly and concisely using only the information available in the context."
+    #          "Do not include any other information which is not there in the context."
+    #     )
 
     try:
         response = openai.chat.completions.create( # Use the new chat completions API

retrieval.py

@@ -3,23 +3,18 @@ import numpy as np
 from langchain.schema import Document
 import faiss
 from rank_bm25 import BM25Okapi
-from data_processing import embedding_model #, index, actual_docs
+from data_processing import embedding_model
 from sentence_transformers import CrossEncoder
 
 reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
 
 retrieved_docs = None
-global query_dataset
-query_dataset = ''
 
-def retrieve_documents_hybrid(query, top_k=5):
-    #global query_dataset
-    query_dataset = find_query_dataset(query)
-    
-    with open( f"data_local/{query_dataset}_chunked_docs.json", "r") as f:
+def retrieve_documents_hybrid(query, q_dataset, top_k=5):
+    with open( f"data_local/{q_dataset}_chunked_docs.json", "r") as f:
         chunked_documents = json.load(f)  # Contains all documents for this dataset
     
-    faiss_index_path = f"data_local/{query_dataset}_quantized.faiss"
+    faiss_index_path = f"data_local/{q_dataset}_quantized.faiss"
     index = faiss.read_index(faiss_index_path)
 
     # Tokenize documents for BM25
@@ -47,23 +42,23 @@ def retrieve_documents_hybrid(query, top_k=5):
     return reranked_docs
 
 # Retrieval Function
-def retrieve_documents(query, top_k=5):
-    query_dataset = find_query_dataset(query)
-    #index, chunk_docs = load_data_from_faiss(query)
+# def retrieve_documents(query, top_k=5):
+#     query_dataset = find_query_dataset(query)
+#     #index, chunk_docs = load_data_from_faiss(query)
 
-    with open( f"data_local/{query_dataset}_chunked_docs.json", "r") as f:
-        documents = json.load(f)  # Contains all documents for this dataset
+#     with open( f"data_local/{query_dataset}_chunked_docs.json", "r") as f:
+#         documents = json.load(f)  # Contains all documents for this dataset
 
-    faiss_index_path = f"data_local/{query_dataset}_quantized.faiss"
-    index = faiss.read_index(faiss_index_path)
+#     faiss_index_path = f"data_local/{query_dataset}_quantized.faiss"
+#     index = faiss.read_index(faiss_index_path)
 
-    query_embedding = np.array(embedding_model.embed_documents([query]), dtype=np.float32)
+#     query_embedding = np.array(embedding_model.embed_documents([query]), dtype=np.float32)
 
-    _, nearest_indices = index.search(query_embedding, top_k)
+#     _, nearest_indices = index.search(query_embedding, top_k)
 
-    retrieved_docs = [Document(page_content=documents[i]) for i in nearest_indices[0]]
+#     retrieved_docs = [Document(page_content=documents[i]) for i in nearest_indices[0]]
 
-    return retrieved_docs
+#     return retrieved_docs
 
 def remove_duplicate_documents(documents):
     unique_documents = []