finalize Gradio updates

app.py

@@ -0,0 +1,145 @@
+import os
+import json
+import gradio as gr
+from pathlib import Path
+from typing import List, Tuple
+from chatbot_config import ChatbotConfig
+from chatbot_model import RetrievalChatbot
+from tf_data_pipeline import TFDataPipeline
+from response_quality_checker import ResponseQualityChecker
+from environment_setup import EnvironmentSetup
+from sentence_transformers import SentenceTransformer
+from logger_config import config_logger
+
+logger = config_logger(__name__)
+
+def load_pipeline():
+    """
+    Loads config, FAISS index, response pool, SentenceTransformer, TFDataPipeline, and sets up the chatbot.
+    """
+    MODEL_DIR = "models"
+    FAISS_INDICES_DIR = os.path.join(MODEL_DIR, "faiss_indices")
+    FAISS_INDEX_PRODUCTION_PATH = os.path.join(FAISS_INDICES_DIR, "faiss_index_production.index")
+    RESPONSE_POOL_PATH = FAISS_INDEX_PRODUCTION_PATH.replace(".index", "_responses.json")
+
+    config_path = Path(MODEL_DIR) / "config.json"
+    if config_path.exists():
+        with open(config_path, "r", encoding="utf-8") as f:
+            config_dict = json.load(f)
+        config = ChatbotConfig.from_dict(config_dict)
+    else:
+        config = ChatbotConfig()
+
+    # Initialize environment
+    env = EnvironmentSetup()
+    env.initialize()
+
+    # Load models and data
+    encoder = SentenceTransformer(config.pretrained_model)
+    
+    data_pipeline = TFDataPipeline(
+        config=config,
+        tokenizer=encoder.tokenizer,
+        encoder=encoder,
+        response_pool=[],
+        query_embeddings_cache={},
+        index_type='IndexFlatIP',
+        faiss_index_file_path=FAISS_INDEX_PRODUCTION_PATH
+    )
+
+    # Load FAISS index and response pool
+    if os.path.exists(FAISS_INDEX_PRODUCTION_PATH) and os.path.exists(RESPONSE_POOL_PATH):
+        data_pipeline.load_faiss_index(FAISS_INDEX_PRODUCTION_PATH)
+        with open(RESPONSE_POOL_PATH, "r", encoding="utf-8") as f:
+            data_pipeline.response_pool = json.load(f)
+        data_pipeline.validate_faiss_index()
+    else:
+        logger.warning("FAISS index or responses are missing. The chatbot may not work properly.")
+
+    chatbot = RetrievalChatbot.load_model(load_dir=MODEL_DIR, mode="inference")
+    quality_checker = ResponseQualityChecker(data_pipeline=data_pipeline)
+
+    return chatbot, quality_checker
+
+# Load the chatbot and quality checker globally
+chatbot, quality_checker = load_pipeline()
+
+def respond(message: str, history: List[List[str]]) -> Tuple[str, List[List[str]]]:
+    """Generate chatbot response using internal context handling."""
+    if not message.strip():  # Skip
+        return "", history
+    
+    try:
+        response, _, metrics, confidence = chatbot.chat(
+            query=message,
+            conversation_history=None,  # Handled internally
+            quality_checker=quality_checker,
+            top_k=10
+        )
+        
+        history.append((message, response))
+        return "", history
+    except Exception as e:
+        logger.error(f"Error generating response: {e}")
+        error_message = "I apologize, but I encountered an error processing your request."
+        history.append((message, error_message))
+        return "", history
+
+def main():
+    """Initialize and launch Gradio interface."""
+    with gr.Blocks(
+        title="Chatbot Demo",
+        css="""
+            .message-wrap { max-height: 800px !important; }
+            .chatbot { min-height: 600px; }
+        """
+    ) as demo:
+        gr.Markdown(
+            """
+            # Retrieval-Based Chatbot Demo using Sentence Transformers + FAISS
+            Knowledge areas: restaurants, movie tickets, rideshare, coffee, pizza, and auto repair.
+            """
+        )
+
+        # Chat interface with custom height
+        chatbot = gr.Chatbot(
+            label="Conversation",
+            container=True,
+            height=600,
+            show_label=True,
+            elem_classes="chatbot"
+        )
+
+        # Input area with send button
+        with gr.Row():
+            msg = gr.Textbox(
+                show_label=False,
+                placeholder="Type your message here...",
+                container=False,
+                scale=8
+            )
+            send = gr.Button(
+                "Send",
+                variant="primary",
+                scale=1,
+                min_width=100
+            )
+
+        clear = gr.Button("Clear Conversation", variant="secondary")
+
+        # Event handlers
+        msg.submit(respond, [msg, chatbot], [msg, chatbot], queue=False)
+        send.click(respond, [msg, chatbot], [msg, chatbot], queue=False)
+        clear.click(lambda: ([], []), outputs=[chatbot, msg], queue=False)
+
+        # Responsive interface
+        msg.change(lambda: None, None, None, queue=False)
+
+    return demo
+
+if __name__ == "__main__":
+    demo = main()
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+    )

readme.md

@@ -1,42 +1,18 @@
-# Retrieval-based learning chatbot
+# CSC252 Retrieval Chatbot
 
-CSC525 - Module 8 Option 2 - Retrieval-based Learning Chatbot - Joseph Armani
+This is a retrieval-based chatbot using Sentence Transformers and FAISS for efficient similarity search.
 
-## TODO
-
-A Python tool to generate high-quality dialog variations.
-
-This package automatically downloads the following models during installation:
-
-- Universal Sentence Encoder v4 (TensorFlow Hub)
-- ChatGPT Paraphraser T5-base
-- Helsinki-NLP translation models (en-de, de-es, es-en)
-- spaCy en_core_web_sm, eng_core_web_md
-- nltk wordnet and averaged_perceptron_tagger_eng models
-
-## Install package
+## Description
 
-pip install -e .
+The chatbot uses a pre-trained Sentence Transformer model to encode queries and a FAISS index to retrieve relevant responses from a curated response pool (Taskmaster-1 dataset)
 
-## Description
+## Usage
 
-This Python script demonstrates a complete pipeline for dialogue augmentation, including validation, optimization, and data augmentation.
-It creates high-quality augmented versions of dialogues by applying various text augmentation techniques and quality control checks.
-Two approaches are used for text augmentation: paraphrasing and back-translation. The pipeline also includes quality metrics for evaluating the augmented text.
-Special handling is implemented for very short text such as greetings and farewells, which are predefined and filtered for quality.
-The pipeline is designed to process a dataset of dialogues and generate multiple high-quality augmented versions of each dialogue.
-The pipeline ensures duplicate dialogues are not generated and that the output meets quality thresholds for semantic similarity, grammar, fluency, diversity, and content preservation.
+Simply type your question in the chat interface and the bot will retrieve the most relevant response from its knowledge base.
+Features
 
-## References
+## Semantic search using Sentence Transformers
 
-Accsany, P. (2024). Working with JSON data in Python. Real Python. <https://realpython.com/python-json/>
-Explosion AI Team. (n.d.). Spacy · industrial-strength natural language processing in python. <https://spacy.io/>
-GeeksforGeeks. (2024). Text augmentation techniques in NLP. GeeksforGeeks. <https://www.geeksforgeeks.org/text-augmentation-techniques-in-nlp/>
-Helsinki-NLP. (2024). Opus-MT [Computer software]. GitHub. <https://github.com/Helsinki-NLP/Opus-MT>
-Hugging Face. (n.d.). Transformers. Hugging Face. <https://huggingface.co/docs/transformers/en/index>
-Humarin. (2023). ChatGPT paraphraser on T5-base [Computer software]. Hugging Face. <https://huggingface.co/humarin/chatgpt_paraphraser_on_T5_base>
-Keita, Z. (2022). Data augmentation in NLP using back-translation with MarianMT. Towards Data Science. <https://towardsdatascience.com/data-augmentation-in-nlp-using-back-translation-with-marianmt-a8939dfea50a>
-Memgraph. (2023). Cosine similarity in Python with scikit-learn. Memgraph. <https://memgraph.com/blog/cosine-similarity-python-scikit-learn>
-Morris, J. (n.d.). language-tool-python (Version 2.8.1) [Computer software]. PyPI. <https://pypi.org/project/language-tool-python/>
-TensorFlow. (n.d.). Universal sentence encoder. TensorFlow Hub. <https://www.tensorflow.org/hub/tutorials/semantic_similarity_with_tf_hub_universal_encoder>
-Waheed, A. (2023). How to calculate ROUGE score in Python. Python Code. <https://thepythoncode.com/article/calculate-rouge-score-in-python>
+Efficient retrieval using FAISS indexing
+Context-aware responses
+Quality checking of responses

requirements.txt

@@ -1,27 +1,29 @@
-faiss-cpu>=1.7.0  # Required for Facebook AI Similarity Search
-h5py>=3.1.0  # For saving and loading models
-ipython>=8.0.0  # For interactive Python
-loguru>=0.7.0  # Enhanced logging (optional but recommended)
-matplotlib>=3.5.0  # For validation plotting
-nlpaug>=1.1.0  # Data augmentation for NLP
-nltk>=3.6.0  # Natural language toolkit
-numpy>=1.19.0  # General numerical computation
-pandas>=1.5.0  # For data handling
-pyyaml>=6.0.0  # For config management
-scikit-learn>=1.0.0  # Machine learning tools
-sacremoses>=0.0.53   # Required for some HuggingFace models
-sentencepiece>=0.1.99  # Required for HuggingFace transformers
-spacy>=3.0.0  # Text processing and tokenization
-tensorflow>=2.13.0  # TensorFlow, for deep learning
-tensorflow-hub>=0.12.0  # Pretrained model hub for TensorFlow
-tokenizers>=0.13.0   # Required for HuggingFace transformers
-torch>=2.0.0  # PyTorch, for deep learning
-tqdm>=4.64.0  # Progress bar
-transformers>=4.30.0  # Hugging Face Transformers library
-typing-extensions>=4.0.0  # For better type hints
+faiss-cpu>=1.7.0        # Facebook AI Similarity Search
+gradio>=3.30.0          # Web app creation
+h5py>=3.1.0             # For saving and loading models
+ipython>=8.0.0          # Interactive Python
+loguru>=0.7.0           # Enhanced logging (optional but recommended)
+matplotlib>=3.5.0       # Validation plotting
+nlpaug>=1.1.0           # Data augmentation for NLP
+nltk>=3.6.0             # Natural language toolkit
+numpy>=1.19.0           # Numerical computation
+pandas>=1.5.0           # Data handling
+pyyaml>=6.0.0           # Config management
+scikit-learn>=1.0.0     # ML tools
+sacremoses>=0.0.53      # Required for some HuggingFace pipelines
+sentencepiece>=0.1.99   # Required for Transformers
+sentence-transformers>=2.2.2  # Sentence embeddings
+spacy>=3.0.0            # Text processing, tokenization
+tensorflow>=2.13.0      # TensorFlow
+tensorflow-hub>=0.12.0  # Pretrained model hub
+tokenizers>=0.13.0      # HuggingFace tokenizers
+torch>=2.0.0            # PyTorch
+tqdm>=4.64.0            # Progress bars
+transformers>=4.30.0    # Hugging Face Transformers
+typing-extensions>=4.0.0
 
-# Dev dependencies
-black>=22.0.0  # For code formatting
-isort>=5.10.0  # For import sorting
-mypy>=1.0.0  # For type checking
-pytest>=7.0.0  # For testing
+# Dev dependencies:
+black>=22.0.0           # Code formatting
+isort>=5.10.0           # Import sorting
+mypy>=1.0.0             # Type checking
+pytest>=7.0.0           # Testing