Issue #35

app/app.py

@@ -10,7 +10,7 @@ from fastapi.responses import JSONResponse
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.triggers.cron import CronTrigger
 
-from routes import category, summary, keyword, lda # pylint: disable=import-error
+from routes import category, summary, keyword, lda, knowledge_base # pylint: disable=import-error
 
 
 class Config: # pylint: disable=too-few-public-methods
@@ -63,7 +63,7 @@ async def lifespan(_app: FastAPI):
     logging.basicConfig(
         format='%(asctime)s - %(levelname)s - %(funcName)s - %(message)s'
     )
-    logging.getLogger().setLevel(logging.INFO)
+    logging.getLogger().setLevel(logging.ERROR)
 
     #start scheduler anf add jobs
     scheduler.start()
@@ -172,6 +172,7 @@ app.include_router(category.router)
 app.include_router(summary.router)
 app.include_router(keyword.router)
 app.include_router(lda.router)
+app.include_router(knowledge_base.router)
 
 @app.get("/_health")
 def health():

app/jobs.json

@@ -32,7 +32,7 @@
         "func": "collectors.finfast.lda:collect",
         "trigger": "cron",
         "hour": 23,
-        "minute": 30
+        "minute": 45
     }
 
 ]

app/models/database/__init__.py

@@ -1,6 +1,10 @@
 """Module for Mongodb database"""
 from .mongodb import (article_collection, entity_collection, category_collection,
     keywords_collection, summary_collection, lda_collection)
+from .astra import KnowledgeBase
+
+# Initialize KnowledgeBase instance
+knowledge_base = KnowledgeBase()
 
 __all__ = [
     "article_collection",
@@ -8,5 +12,6 @@ __all__ = [
     "category_collection",
     "keywords_collection",
     "summary_collection",
-    "lda_collection"
+    "lda_collection",
+    "knowledge_base"
 ]

app/models/database/astra.py

@@ -0,0 +1,78 @@
+"""Module for AstraDB database"""
+import logging
+import os
+
+import astrapy
+from dotenv import load_dotenv
+
+load_dotenv()
+
+logging.basicConfig(
+    format='%(asctime)s - %(levelname)s - %(funcName)s - %(message)s',
+    datefmt="%Y-%m-%d %H:%M:%S",
+    level=logging.ERROR)
+
+
+class KnowledgeBase:
+    """
+    AstraDB class for direct collection operations.
+    """
+
+    def __init__(self):
+        """Initialize AstraDB connection."""
+        self.collection = astrapy.DataAPIClient(
+            os.environ["ASTRA_DB_APPLICATION_TOKEN"]).get_database(
+                os.environ["ASTRA_DB_API_ENDPOINT"]).documents
+
+    def get_doc_count(self, user_id: str) -> dict:
+        """
+        Count unique emails and files for a specific user.
+        
+        Args:
+            user_id (str): The user's email address
+            
+        Returns:
+            dict: {"emails": count, "files": count, "total_documents": count}
+            
+        Raises:
+            ValueError: If user_id is invalid
+            Exception: If database query fails
+        """
+        if not user_id or not isinstance(user_id, str):
+            raise ValueError("user_id must be a non-empty string")
+
+        try:
+            # Get all documents for the user with type gmail or file
+            filter_criteria = {
+                "metadata.userId": user_id,
+                "metadata.type": {"$in": ["gmail", "file"]}
+            }
+
+            # Use direct collection access
+            results = list(self.collection.find(filter=filter_criteria))
+
+            # Group by metadata.id to get unique documents
+            unique_docs = {}
+            for doc in results:
+                doc_id = doc.get("metadata", {}).get("id")
+                doc_type = doc.get("metadata", {}).get("type")
+
+                if doc_id and doc_type:
+                    if doc_id not in unique_docs:
+                        unique_docs[doc_id] = doc_type
+
+            # Count by type
+            email_count = sum(1 for doc_type in unique_docs.values() if doc_type == "gmail")
+            file_count = sum(1 for doc_type in unique_docs.values() if doc_type == "file")
+            total_count = len(unique_docs)
+
+            return {
+                "emails": email_count,
+                "files": file_count,
+                "total_documents": total_count
+            }
+
+        except Exception as e:  # pylint: disable=broad-exception-caught
+            logging.error("Failed to get document count for user %s: %s", user_id, str(e))
+            # pylint: disable=raise-missing-from
+            raise Exception(f"Database query failed: {str(e)}") # pylint: disable=broad-exception-raised

app/requirements.txt

@@ -31,6 +31,7 @@ boto3==1.38.43
 nltk==3.9.1
 langchain==0.3.27
 langchain-openai==0.3.28
+langchain-astradb==0.6.0
 gensim==4.3.3
 pyLDAvis==3.4.1
 scikit-learn==1.7.1

app/routes/knowledge_base.py

@@ -0,0 +1,59 @@
+"""This module defines the /knowledge-base route for the FastAPI application."""
+import logging
+from fastapi import APIRouter, HTTPException, Path
+from fastapi.responses import JSONResponse
+from models.database import knowledge_base  # pylint: disable=import-error
+
+router = APIRouter(prefix="/knowledge-base", tags=["knowledge-base"])
+
+@router.get("/{user_id}")
+async def get_user_document_stats(
+    user_id: str = Path(..., description="User's email address")
+    ) -> JSONResponse:
+    """
+    Get document statistics for a specific user.
+    
+    This endpoint counts the number of unique emails and files uploaded by the user.
+    It groups documents by metadata.id to ensure unique document counting (not chunks).
+    
+    Args:
+        user_id (str): The user's email address
+        
+    Returns:
+        JSONResponse: A JSON response containing document counts:
+        {
+            "user_id": "[email protected]",
+            "emails": 5,
+            "files": 12,
+            "total_documents": 17
+        }
+        
+    Raises:
+        HTTPException: 400 for invalid user_id, 500 for database errors
+    """
+    try:
+        # Validate user_id format (basic email validation)
+        if not user_id or "@" not in user_id or "." not in user_id:
+            raise HTTPException(
+                status_code=400,
+                detail="Invalid user_id format. Must be a valid email address."
+            )
+
+        # Get document counts from database
+        result = knowledge_base.get_doc_count(user_id)
+
+        # Add user_id to response
+        result["user_id"] = user_id
+
+        return JSONResponse(content=result, status_code=200)
+
+    except ValueError as e:
+        logging.error("Validation error for user %s: %s", user_id, str(e))
+        raise HTTPException(status_code=400, detail=str(e)) # pylint: disable=raise-missing-from
+
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logging.error("Database error for user %s: %s", user_id, str(e))
+        raise HTTPException( # pylint: disable=raise-missing-from
+            status_code=500,
+            detail="Internal server error while retrieving document statistics."
+        )