Use cross encoder for QA

app.py

@@ -1,7 +1,7 @@
 import streamlit as st
 import pandas as pd
 from transformers import pipeline
-from sklearn.metrics.pairwise import cosine_similarity
+from sentence_transformers import CrossEncoder
 from sentence_transformers import SentenceTransformer
 import string
 from nltk.tokenize import word_tokenize
@@ -99,8 +99,8 @@ st.markdown("<div class='custom-header'> 🧩 AI-Powered News Analyzer</div>", u
 classifier = pipeline("text-classification", model="Sandini/news-classifier")  # Classification pipeline
 qa_pipeline = pipeline("question-answering", model="distilbert/distilbert-base-cased-distilled-squad")  # QA pipeline
 
-# Initialize sentence transformer model for QA similarity
-sentence_model = SentenceTransformer('all-MiniLM-L6-v2')  # Pre-trained sentence model
+# Initialize Cross-Encoder for QA relevance scoring
+cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')  # Pre-trained Cross-Encoder model
 
 # Define preprocessing functions for classification
 def preprocess_text(text):
@@ -200,15 +200,14 @@ with col2:
             if 'content' in df.columns:
                 context = df['content'].dropna().tolist()  # Use the content column as context
 
-                # Generate embeddings for the context and the question
-                context_embeddings = sentence_model.encode(context)
-                question_embedding = sentence_model.encode([user_question])
+                # Prepare pairs of (question, context)
+                pairs = [(user_question, c) for c in context]
 
-                # Calculate cosine similarity
-                similarities = cosine_similarity(question_embedding, context_embeddings)
-                top_indices = similarities[0].argsort()[-5:][::-1]  # Get top 5 similar rows
+                # Score each pair using the Cross-Encoder
+                scores = cross_encoder.predict(pairs)
 
-                # Prepare the top 5 similar context rows
+                # Get top matches based on scores
+                top_indices = scores.argsort()[-5:][::-1]  # Get indices of top 5 matches
                 top_context = "\n".join([context[i] for i in top_indices])
 
                 # Get answer from Hugging Face model using top context