Merge pull request #43 from oxbridge-econ/dev

app/app.py

@@ -10,7 +10,7 @@ from fastapi.responses import JSONResponse
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from apscheduler.triggers.cron import CronTrigger
 
-from routes import category, summary, keyword, lda, knowledge_base # pylint: disable=import-error
+from routes import category, summary, keyword, lda, entity # pylint: disable=import-error
 
 
 class Config: # pylint: disable=too-few-public-methods
@@ -172,7 +172,7 @@ app.include_router(category.router)
 app.include_router(summary.router)
 app.include_router(keyword.router)
 app.include_router(lda.router)
-app.include_router(knowledge_base.router)
+app.include_router(entity.router)
 
 @app.get("/_health")
 def health():

app/collectors/finfast/keyword_analysis.py

@@ -140,4 +140,3 @@ def _perform_analysis_for_all_periods() -> None:
         except Exception as e: # pylint: disable=broad-exception-caught
             logger.error("Error analyzing %s period: %s", period, str(e))
             # Continue with other periods even if one fails
- 

app/controllers/entity.py

@@ -0,0 +1,29 @@
+"""Controller for entity-related operations."""
+from models.database import entity_collection
+
+def retrieve_hot_entity():
+    """Retrieves the top 200 documents from a MongoDB collection,
+    sorted by the occurrence field in descending order.
+    The function returns a dict containing the count of documents and a list of documents
+    Parameters
+    None
+
+    Type: dict
+    Structure:
+
+    Count (int): The number of documents retrieved (up to 200).
+    Items (list): A list of dictionaries, each containing:
+
+    entity: The value of the entity field from the document.
+    entityType: The value of the entityType field from the document.
+    total_occurrence: The value of the occurrence field from the document.
+    """
+    result = list(entity_collection.find(
+        {},
+        {'entity': 1, 'entityType': 1, 'total_occurrence': '$occurrence', '_id': 0}
+    ).sort('occurrence', -1).limit(200))
+    res = {
+        "Count": len(result),
+        "Items": result
+    }
+    return res

app/controllers/keyword.py

@@ -661,7 +661,7 @@ def invoke_llm_for_batch(articles_in_batch: list, historical_keywords: set) -> L
 
     try:
         categories_text = "\n".join([
-            f"- {cat}: {', '.join(keywords[:10])}" + ("..." if len(keywords) > 10 else "") 
+            f"- {cat}: {', '.join(keywords[:10])}" + ("..." if len(keywords) > 10 else "")
             for cat, keywords in FIN_KEYWORDS.items()
         ])
 

app/models/database/astra.py

@@ -67,9 +67,9 @@ class KnowledgeBase: # pylint: disable=too-few-public-methods
             total_count = len(unique_docs)
 
             return {
-                "emails": email_count,
-                "files": file_count,
-                "total_documents": total_count
+                "gmail": email_count,
+                "file": file_count,
+                "total": total_count
             }
 
         except Exception as e:  # pylint: disable=broad-exception-caught

app/models/llm/azure.py

@@ -32,4 +32,3 @@ class GPTModel(AzureChatOpenAI):
             streaming=True,
             temperature=0
         )
-       

app/routes/entity.py

@@ -0,0 +1,22 @@
+"""This module defines the /lda route for the FastAPI application."""
+
+import logging
+from fastapi import APIRouter
+from fastapi.responses import JSONResponse
+from controllers.entity import retrieve_hot_entity # pylint: disable=import-error
+
+# Configure logger
+logger = logging.getLogger(__name__)
+
+# Create FastAPI Router
+router = APIRouter(prefix="/entity", tags=["entity"])
+
+@router.get('/hot')
+async def get_hot_entity():
+    """
+    Handles GET requests to retrieve hot entity.
+
+    Returns:
+        dict: JSON response containing count and a list of hot entity.
+    """
+    return JSONResponse(content=retrieve_hot_entity())

app/routes/knowledge_base.py

@@ -1,59 +0,0 @@
-"""This module defines the /knowledge-base route for the FastAPI application."""
-import logging
-from fastapi import APIRouter, HTTPException, Path
-from fastapi.responses import JSONResponse
-from models.database import knowledge_base  # pylint: disable=import-error
-
-router = APIRouter(prefix="/knowledge-base", tags=["knowledge-base"])
-
-@router.get("/{user_id}")
-async def get_user_document_stats(
-    user_id: str = Path(..., description="User's email address")
-    ) -> JSONResponse:
-    """
-    Get document statistics for a specific user.
-    
-    This endpoint counts the number of unique emails and files uploaded by the user.
-    It groups documents by metadata.id to ensure unique document counting (not chunks).
-    
-    Args:
-        user_id (str): The user's email address
-        
-    Returns:
-        JSONResponse: A JSON response containing document counts:
-        {
-            "user_id": "[email protected]",
-            "emails": 5,
-            "files": 12,
-            "total_documents": 17
-        }
-        
-    Raises:
-        HTTPException: 400 for invalid user_id, 500 for database errors
-    """
-    try:
-        # Validate user_id format (basic email validation)
-        if not user_id or "@" not in user_id or "." not in user_id:
-            raise HTTPException(
-                status_code=400,
-                detail="Invalid user_id format. Must be a valid email address."
-            )
-
-        # Get document counts from database
-        result = knowledge_base.get_doc_count(user_id)
-
-        # Add user_id to response
-        result["user_id"] = user_id
-
-        return JSONResponse(content=result, status_code=200)
-
-    except ValueError as e:
-        logging.error("Validation error for user %s: %s", user_id, str(e))
-        raise HTTPException(status_code=400, detail=str(e)) # pylint: disable=raise-missing-from
-
-    except Exception as e:  # pylint: disable=broad-exception-caught
-        logging.error("Database error for user %s: %s", user_id, str(e))
-        raise HTTPException( # pylint: disable=raise-missing-from
-            status_code=500,
-            detail="Internal server error while retrieving document statistics."
-        )

app/routes/summary.py

@@ -1,12 +1,67 @@
 """This module defines the /summary route for the Flask application."""
 
 import importlib
-from fastapi import APIRouter
+from venv import logger
+import re
+from fastapi import APIRouter, HTTPException, Path
 from fastapi.responses import JSONResponse
 from controllers.summary import get_summary_data # pylint: disable=import-error
+from models.database import knowledge_base  # pylint: disable=import-error
 
 router = APIRouter(prefix="/summary", tags=["summary"])
 
+@router.get("/{email}")
+async def get_user_document_stats(
+    email: str = Path(..., description="User's email address")
+    ) -> JSONResponse:
+    """
+    Get document statistics for a specific user.
+    
+    This endpoint counts the number of unique emails and files uploaded by the user.
+    It groups documents by metadata.id to ensure unique document counting (not chunks).
+    
+    Args:
+        email (str): The user's email address
+        
+    Returns:
+        JSONResponse: A JSON response containing document counts:
+        {
+            "email": "[email protected]",
+            "emails": 5,
+            "files": 12,
+            "total_documents": 17
+        }
+        
+    Raises:
+        HTTPException: 400 for invalid email, 500 for database errors
+    """
+    try:
+        email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+        if not email or not re.match(email_pattern, email):
+            raise HTTPException(
+                status_code=400,
+                detail="Invalid email format. Must be a valid email address."
+            )
+
+        # Get document counts from database
+        result = knowledge_base.get_doc_count(email)
+
+        # Add email to response
+        result["email"] = email
+
+        return JSONResponse(content=result, status_code=200)
+
+    except ValueError as e:
+        logger.error("Validation error for user %s: %s", email, str(e))
+        raise HTTPException(status_code=400, detail=str(e)) # pylint: disable=raise-missing-from
+
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logger.error("Database error for user %s: %s", email, str(e))
+        raise HTTPException( # pylint: disable=raise-missing-from
+            status_code=500,
+            detail="Internal server error while retrieving document statistics."
+        )
+
 @router.get('')
 async def get_summary() -> JSONResponse:
     """
@@ -31,7 +86,6 @@ async def get_summary() -> JSONResponse:
     except Exception as e: #pylint: disable=broad-except
         return JSONResponse(content={"error": str(e)}, status_code=500)
 
-
 @router.get("/{module}/{chart_id}")
 def get_summary_chart(module: str, chart_id: str) -> JSONResponse:
     """
@@ -59,7 +113,6 @@ def get_summary_chart(module: str, chart_id: str) -> JSONResponse:
     except AttributeError as e:
         return JSONResponse(content={"error": str(e)}, status_code=500)
 
-
 @router.get("/{module}")
 async def get_summary_module(module: str) -> JSONResponse:
     """
@@ -79,3 +132,55 @@ async def get_summary_module(module: str) -> JSONResponse:
         return JSONResponse(content={"error": str(e)}, status_code=404)
     except Exception as e: #pylint: disable=broad-except
         return JSONResponse(content={"error": str(e)}, status_code=500)
+
+# @router.get("/{email}")
+# async def get_user_document_stats(
+#     email: str = Path(..., description="User's email address")
+#     ) -> JSONResponse:
+#     """
+#     Get document statistics for a specific user.
+    
+#     This endpoint counts the number of unique emails and files uploaded by the user.
+#     It groups documents by metadata.id to ensure unique document counting (not chunks).
+    
+#     Args:
+#         email (str): The user's email address
+        
+#     Returns:
+#         JSONResponse: A JSON response containing document counts:
+#         {
+#             "email": "[email protected]",
+#             "emails": 5,
+#             "files": 12,
+#             "total_documents": 17
+#         }
+        
+#     Raises:
+#         HTTPException: 400 for invalid email, 500 for database errors
+#     """
+#     try:
+#         email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
+#         if not email or not re.match(email_pattern, email):
+#             raise HTTPException(
+#                 status_code=400,
+#                 detail="Invalid email format. Must be a valid email address."
+#             )
+
+#         # Get document counts from database
+#         result = knowledge_base.get_doc_count(email)
+
+#         # Add email to response
+#         result["email"] = email
+
+#         return JSONResponse(content=result, status_code=200)
+
+#     except ValueError as e:
+#         logger.error("Validation error for user %s: %s", email, str(e))
+#         raise HTTPException(status_code=400, detail=str(e)) # pylint: disable=raise-missing-from
+
+#     except Exception as e:  # pylint: disable=broad-exception-caught
+#         logger.error("Database error for user %s: %s", email, str(e))
+#         raise HTTPException( # pylint: disable=raise-missing-from
+#             status_code=500,
+#             detail="Internal server error while retrieving document statistics."
+#         )