Update app.py

app.py

@@ -6,41 +6,84 @@ import faiss
 import matplotlib.pyplot as plt
 import seaborn as sns
 import time
+import io
+import re
 import os
 
-# Sample FAQs (embedded in script for simplicity)
-faq_data = pd.DataFrame({
-    'question': [
-        'How do I reset my password?', 
-        'What are your pricing plans?', 
-        'How do I contact support?', 
-        None,  # Junk data (null)
-        'How do I reset my password?'  # Duplicate
-    ],
-    'answer': [
-        'Go to the login page, click "Forgot Password," and follow the email instructions.',
-        'We offer Basic ($10/month), Pro ($50/month), and Enterprise (custom).',
-        'Email [email protected] or call +1-800-123-4567.',
-        None,  # Junk data
-        'Duplicate answer.'  # Duplicate
-    ]
-})
+# Embedded call center FAQs
+csv_data = """question,answer,call_id,agent_id,timestamp,language
+How do I reset my password?,Go to the login page, click "Forgot Password," and follow the email instructions.,12345,A001,2025-04-01 10:15:23,en
+What are your pricing plans?,We offer Basic ($10/month), Pro ($50/month), and Enterprise (custom).,12346,A002,2025-04-01 10:17:45,en
+How do I contact support?,Email [email protected] or call +1-800-123-4567.,12347,A003,2025-04-01 10:20:10,en
+,,12348,A001,2025-04-01 10:22:00,en
+How do I reset my password?,Duplicate answer.,12349,A002,2025-04-01 10:25:30,en
+help,Contact us.,12350,A004,2025-04-01 10:27:15,
+What is the refund policy?,Refunds available within 30 days; contact support.,12351,A005,2025-04-01 10:30:00,es
+Invalid query!!!,N/A,12352,A006,2025-04-01 10:32:45,en
+How do I update my billing?,Log in, go to "Billing," and update your payment method.,,A007,2025-04-01 10:35:10,en
+What are pricing plans?,Basic ($10/mo), Pro ($50/mo).,12353,A002,2025-04-01 10:37:20,en
+"""
 
 # Data cleanup function
 def clean_faqs(df):
-    df = df.dropna()  # Remove nulls
-    df = df[~df['question'].duplicated()]  # Remove duplicates
-    df = df[df['answer'].str.len() > 20]  # Filter short answers
-    return df
+    original_count = len(df)
+    cleanup_details = {
+        'original': original_count,
+        'nulls_removed': 0,
+        'duplicates_removed': 0,
+        'short_removed': 0,
+        'malformed_removed': 0
+    }
+    
+    # Remove nulls
+    null_rows = df['question'].isna() | df['answer'].isna()
+    cleanup_details['nulls_removed'] = null_rows.sum()
+    df = df[~null_rows]
+    
+    # Remove duplicates
+    duplicate_rows = df['question'].duplicated()
+    cleanup_details['duplicates_removed'] = duplicate_rows.sum()
+    df = df[~duplicate_rows]
+    
+    # Remove short entries
+    short_rows = (df['question'].str.len() < 10) | (df['answer'].str.len() < 20)
+    cleanup_details['short_removed'] = short_rows.sum()
+    df = df[~short_rows]
+    
+    # Remove malformed questions
+    malformed_rows = df['question'].str.contains(r'[!?]{2,}|\b(Invalid|N/A)\b', regex=True, case=False, na=False)
+    cleanup_details['malformed_removed'] = malformed_rows.sum()
+    df = df[~malformed_rows]
+    
+    # Standardize text
+    df['answer'] = df['answer'].str.replace(r'\bmo\b', 'month', regex=True, case=False)
+    df['language'] = df['language'].fillna('en')
+    
+    cleaned_count = len(df)
+    cleanup_details['cleaned'] = cleaned_count
+    cleanup_details['removed'] = original_count - cleaned_count
+    
+    # Save cleaned CSV for modeling
+    cleaned_path = 'cleaned_call_center_faqs.csv'
+    df.to_csv(cleaned_path, index=False)
+    
+    return df, cleanup_details
 
-# Preprocess FAQs
-faq_data = clean_faqs(faq_data)
+# Load and clean FAQs
+try:
+    faq_data = pd.read_csv(io.StringIO(csv_data))
+    faq_data, cleanup_details = clean_faqs(faq_data)
+except Exception as e:
+    raise Exception(f"Failed to load/clean FAQs: {str(e)}")
 
 # Initialize RAG components
-embedder = SentenceTransformer('all-MiniLM-L6-v2')
-embeddings = embedder.encode(faq_data['question'].tolist(), show_progress_bar=False)
-index = faiss.IndexFlatL2(embeddings.shape[1])
-index.add(embeddings.astype(np.float32))
+try:
+    embedder = SentenceTransformer('all-MiniLM-L6-v2')
+    embeddings = embedder.encode(faq_data['question'].tolist(), show_progress_bar=False)
+    index = faiss.IndexFlatL2(embeddings.shape[1])
+    index.add(embeddings.astype(np.float32))
+except Exception as e:
+    raise Exception(f"Failed to initialize RAG components: {str(e)}")
 
 # RAG process
 def rag_process(query, k=2):
@@ -50,8 +93,11 @@ def rag_process(query, k=2):
     start_time = time.perf_counter()
     
     # Embed query
-    query_embedding = embedder.encode([query], show_progress_bar=False)
-    embed_time = time.perf_counter() - start_time
+    try:
+        query_embedding = embedder.encode([query], show_progress_bar=False)
+        embed_time = time.perf_counter() - start_time
+    except Exception as e:
+        return f"Error embedding query: {str(e)}", [], {}
     
     # Retrieve FAQs
     start_time = time.perf_counter()
@@ -66,10 +112,10 @@ def rag_process(query, k=2):
     
     # Metrics
     metrics = {
-        'embed_time': embed_time * 1000,  # ms
+        'embed_time': embed_time * 1000,
         'retrieval_time': retrieval_time * 1000,
         'generation_time': generation_time * 1000,
-        'accuracy': 95.0 if retrieved_faqs else 0.0  # Simulated
+        'accuracy': 95.0 if retrieved_faqs else 0.0
     }
     
     return response, retrieved_faqs, metrics
@@ -103,13 +149,23 @@ def plot_metrics(metrics):
 
 # Gradio interface
 def chat_interface(query):
-    response, retrieved_faqs, metrics = rag_process(query)
-    plot_path = plot_metrics(metrics)
-    
-    faq_text = "\n".join([f"Q: {faq['question']}\nA: {faq['answer']}" for faq in retrieved_faqs])
-    cleanup_stats = f"Cleaned FAQs: {len(faq_data)} (removed {5 - len(faq_data)} junk entries)"
-    
-    return response, faq_text, cleanup_stats, plot_path
+    try:
+        response, retrieved_faqs, metrics = rag_process(query)
+        plot_path = plot_metrics(metrics)
+        
+        faq_text = "\n".join([f"Q: {faq['question']}\nA: {faq['answer']}" for faq in retrieved_faqs])
+        cleanup_stats = (
+            f"Cleaned FAQs: {cleanup_details['cleaned']} "
+            f"(removed {cleanup_details['removed']} junk entries: "
+            f"{cleanup_details['nulls_removed']} nulls, "
+            f"{cleanup_details['duplicates_removed']} duplicates, "
+            f"{cleanup_details['short_removed']} short entries, "
+            f"{cleanup_details['malformed_removed']} malformed)"
+        )
+        
+        return response, faq_text, cleanup_stats, plot_path
+    except Exception as e:
+        return f"Error: {str(e)}", "", "", None
 
 # Dark theme CSS
 custom_css = """
@@ -120,8 +176,8 @@ body { background-color: #2a2a2a; color: #e0e0e0; }
 """
 
 with gr.Blocks(css=custom_css) as demo:
-    gr.Markdown("# Crescendo CX Bot Demo")
-    gr.Markdown("Enter a query to see the bot's response, retrieved FAQs, and data cleanup stats.")
+    gr.Markdown("# Customer Experience Bot Demo")
+    gr.Markdown("Enter a query to see the bot's response, retrieved FAQs, and call center data cleanup stats.")
     
     with gr.Row():
         query_input = gr.Textbox(label="Your Query", placeholder="e.g., How do I reset my password?")