pushing api

.env.template

@@ -0,0 +1 @@
+OPENAI_API_KEY=

.gitignore

@@ -0,0 +1,8 @@
+__pycache__
+*.lock
+.env
+dev
+*.log
+*.csv
+raw/
+vector/

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.11.4
+
+WORKDIR /app
+
+COPY . /app/
+
+RUN apt-get update && \
+    apt-get install -y \
+    redis-server \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN pip install --upgrade pip && \
+    pip install uv && \
+    uv pip install --system -r requirements.txt
+
+CMD ["python", "-m", "main"]

README.md

@@ -1,10 +1,111 @@
----
-title: Eda Franky V1
-emoji: 📊
-colorFrom: green
-colorTo: gray
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Franky EDA Engine
+
+## Project Structure
+
+```plaintext
+FRANKY-API/
+├── knowledge_base/
+│   ├── raw/
+│   └── vector/
+├── src/
+│   ├── api/
+│   │    ├── v1/
+│   │    │    ├── build_ml_plan/
+│   │    │    │      ├── __init__.py
+│   │    │    │      ├── eda.py
+│   │    │    │      └── task_analysis.py
+│   │    │    ├── eda_engine/
+│   │    │    │      ├── __init__.py
+│   │    │    │      ├── data_quality.py
+│   │    │    │      ├── data_statistics.py
+│   │    │    │      ├── data_understanding.py
+│   │    │    │      └── univariate_analysis.py
+│   │    │    └── __init__.py
+│   │    └── __init__.py
+│   ├── app/
+│   │    ├── pipelines/
+│   │    │     ├── eda/
+│   │    │     │     ├── agemts/
+│   │    │     │     │      ├── agents.py
+│   │    │     │     │      ├── models.py
+│   │    │     │     │      └── prompts.py
+│   │    │     │     ├── tools/
+│   │    │     │     │      ├── analysis_tools/
+│   │    │     │     │      │      ├── __init__.py
+│   │    │     │     │      │      ├── bivariate_analysis.py
+│   │    │     │     │      │      ├── multivariate_analysis.py
+│   │    │     │     │      │      └── univariate_analysis.py
+│   │    │     │     │      ├── data_cleaning_tools/
+│   │    │     │     │      │      ├── __init__.py
+│   │    │     │     │      │      ├── handle_missing_values.py
+│   │    │     │     │      │      └── handle_outliers.py
+│   │    │     │     │      ├── transformations_tools/
+│   │    │     │     │      └── lib.py
+│   │    │     │     ├── helper.py
+│   │    │     │     ├── model.py
+│   │    │     │     └── pipeline.py
+│   │    │     ├── modules/
+│   │    │     │     ├── __init__.py
+│   │    │     │     ├── data_quality_assessment.py
+│   │    │     │     ├── data_statistics.py
+│   │    │     │     ├── data_understanding_context.py
+│   │    │     │     └── univariate_analysis.py
+│   │    │     └── task_analysis/
+│   │    │           ├── __init__.py
+│   │    │           ├── ml_analysis_workflow.py
+│   │    │           ├── ml_implementation_planner_workflow.py
+│   │    │           └── model.py
+│   │    └── schemas/
+│   │          ├── requests/
+│   │          │      ├── __init__.py
+│   │          │      ├── eda.py
+│   │          │      └── task_analysis.py
+│   │          └── responses/
+│   │                 └── eda.py
+│   └── core/
+│        ├── cache/
+│        │      ├── code_generated/
+│        │      ├── dataset_logs/
+│        │      ├── downloads/
+│        │      └── redis_cache.py
+│        ├── database/
+│        ├── logs/
+│        ├── utils/
+│        │      ├── __init__.py
+│        │      ├── knowledge_base.py
+│        │      ├── logger.py
+│        │      ├── prompts.py
+│        │      └── read_config.py
+│        └── server.py
+├── .env.template
+├── .gitignore
+├── config.yaml
+├── Dockerfile
+├── main.py
+├── prepare_kb.py
+├── README.md
+└── Dockerfile
+```
+
+## Getting Started
+
+### 1. Clone the Repository
+
+### 2. Running the Docker Container
+
+To build the docker image, run the following cmd:
+
+```bash
+docker build -t franky-api .
+```
+
+To run the docker container, run the following cmd:
+
+```bash
+docker run -d -p 8000:8000 --name franky-api-container franky-api
+```
+
+You can access the api at:
+
+[http://localhost:8000/](http://localhost:8000/docs)
+

config.yaml

@@ -0,0 +1,13 @@
+app:
+  verbose: True
+
+server:
+  host: "127.0.0.1"
+  port: 8000
+  reload: True
+  workers: 1
+
+redis_server:
+  host: "127.0.0.1"
+  port: 6379
+  db: 0

main.py

@@ -0,0 +1,47 @@
+import uvicorn
+import subprocess
+import shutil
+import time
+import asyncio
+from src.core.utils import logger, read_config
+
+def start_redis_server(redis_config: dict):
+    redis_path = shutil.which("redis-server")
+    if not redis_path:
+        raise RuntimeError("redis-server is not installed or not in PATH")
+
+    process = subprocess.Popen(
+        [redis_path, "--port", str(redis_config['port']), "--bind", redis_config['host']],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL
+    )
+
+    time.sleep(1)
+    logger.info(
+        f"Redis server started successfully on {redis_config['host']}:{redis_config['port']}",
+        log_type="server",
+        console=True
+    )
+    return process
+
+def initialize_config() -> dict:
+    return read_config(config_path="config.yaml")
+
+async def main():
+    config = initialize_config()
+    redis_process = start_redis_server(redis_config=config['redis_server'])
+
+    try:
+        uvicorn.run(
+            app="src.core.server:app",
+            host=config['server']['host'],
+            port=config['server']['port'],
+            reload=config['server']['reload'],
+            workers=config['server']['workers']
+        )
+    finally:
+        logger.info("Shutting down Redis server...", log_type="server", console=config['app']['verbose'])
+        redis_process.terminate()
+
+if __name__ == "__main__":
+    asyncio.run(main())

prepare_kb.ipynb

@@ -0,0 +1,118 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "f5a0d75d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import chromadb\n",
+    "from llama_index.core import StorageContext\n",
+    "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
+    "# from llama_index.embeddings.fastembed import FastEmbedEmbedding\n",
+    "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
+    "from llama_index.core import SimpleDirectoryReader, StorageContext, VectorStoreIndex\n",
+    "\n",
+    "# embed_model = FastEmbedEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")\n",
+    "data_dir = r\"knowledge_base\\raw\\classification\"\n",
+    "\n",
+    "documents = SimpleDirectoryReader(str(data_dir)).load_data()\n",
+    "data_path = r\"knowledge_base\\vector\\classification\"\n",
+    "db = chromadb.PersistentClient(path=data_path)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b52b6ba8",
+   "metadata": {},
+   "source": [
+    "### Storing the data locally"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "348df588",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chroma_collection = db.get_or_create_collection(\"classification_db\")\n",
+    "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+    "index = VectorStoreIndex.from_documents(\n",
+    "    documents=documents,\n",
+    "    storage_context=storage_context,\n",
+    "    show_progress=True,\n",
+    "    # embed_model=embed_model\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f7411c03",
+   "metadata": {},
+   "source": [
+    "### Loading the locally stored vector index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "4d9cbd1b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import chromadb\n",
+    "from llama_index.core import StorageContext\n",
+    "from llama_index.core import VectorStoreIndex\n",
+    "from llama_index.core.retrievers import VectorIndexRetriever\n",
+    "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
+    "# from llama_index.embeddings.fastembed import FastEmbedEmbedding\n",
+    "\n",
+    "# embed_model = FastEmbedEmbedding(model_name=\"BAAI/bge-small-en-v1.5\")\n",
+    "\n",
+    "data_path = r\"knowledge_base\\vector\\classification\"\n",
+    "db = chromadb.PersistentClient(path=data_path)\n",
+    "chroma_collection = db.get_or_create_collection(\"classification_db\")\n",
+    "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+    "\n",
+    "index = VectorStoreIndex.from_vector_store(vector_store, storage_context=storage_context)\n",
+    "retriever = VectorIndexRetriever(\n",
+    "    index, \n",
+    "    # embed_model=embed_model\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "05804310",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "dev",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

requirements.txt

@@ -0,0 +1,201 @@
+agno==1.1.3
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.18
+aiosignal==1.3.2
+annotated-types==0.7.0
+anyio==4.9.0
+appnope==0.1.4
+asgiref==3.8.1
+asttokens==3.0.0
+attrs==25.3.0
+backoff==2.2.1
+banks==2.1.2
+bcrypt==4.3.0
+build==1.2.2.post1
+cachetools==5.5.2
+certifi==2025.1.31
+charset-normalizer==3.4.1
+chromadb==1.0.9
+click==8.1.8
+colorama==0.4.6
+coloredlogs==15.0.1
+comm==0.2.2
+contourpy==1.3.1
+cycler==0.12.1
+dataclasses-json==0.6.7
+debugpy==1.8.14
+decorator==5.2.1
+Deprecated==1.2.18
+dirtyjson==1.0.8
+distro==1.9.0
+dnspython==2.7.0
+docling-core==2.28.1
+docstring_parser==0.16
+duckduckgo_search==8.0.0
+durationpy==0.9
+email_validator==2.2.0
+executing==2.2.0
+faiss-cpu==1.11.0
+fastapi==0.115.9
+fastapi-cli==0.0.7
+fastembed==0.6.1
+filelock==3.18.0
+filetype==1.2.0
+flatbuffers==25.2.10
+fonttools==4.57.0
+frozenlist==1.6.0
+fsspec==2025.3.2
+gitdb==4.0.12
+GitPython==3.1.44
+google-auth==2.40.1
+googleapis-common-protos==1.70.0
+greenlet==3.2.1
+griffe==1.7.3
+groq==0.23.1
+grpcio==1.71.0
+h11==0.14.0
+httpcore==1.0.8
+httptools==0.6.4
+httpx==0.28.1
+huggingface-hub==0.30.2
+humanfriendly==10.0
+idna==3.10
+importlib_metadata==8.6.1
+importlib_resources==6.5.2
+ipykernel==6.29.5
+ipython==9.1.0
+ipython_pygments_lexers==1.1.1
+jedi==0.19.2
+Jinja2==3.1.6
+jiter==0.9.0
+joblib==1.4.2
+jsonref==1.1.0
+jsonschema==4.23.0
+jsonschema-specifications==2025.4.1
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+kiwisolver==1.4.8
+kubernetes==32.0.1
+latex2mathml==3.77.0
+llama-index-core==0.12.34.post1
+llama-index-embeddings-fastembed==0.3.1
+llama-index-embeddings-openai==0.3.1
+llama-index-vector-stores-chroma==0.4.1
+llama-index-vector-stores-faiss==0.4.0
+lmoments3==1.0.8
+loguru==0.7.3
+lxml==5.3.2
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+marshmallow==3.26.1
+matplotlib==3.10.1
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+missingno==0.5.2
+mmh3==5.1.0
+mpmath==1.3.0
+multidict==6.4.3
+mypy_extensions==1.1.0
+nest-asyncio==1.6.0
+networkx==3.4.2
+nltk==3.9.1
+numpy==2.2.4
+oauthlib==3.2.2
+onnxruntime==1.21.1
+openai==1.72.0
+opentelemetry-api==1.33.1
+opentelemetry-exporter-otlp-proto-common==1.33.1
+opentelemetry-exporter-otlp-proto-grpc==1.33.1
+opentelemetry-instrumentation==0.54b1
+opentelemetry-instrumentation-asgi==0.54b1
+opentelemetry-instrumentation-fastapi==0.54b1
+opentelemetry-proto==1.33.1
+opentelemetry-sdk==1.33.1
+opentelemetry-semantic-conventions==0.54b1
+opentelemetry-util-http==0.54b1
+orjson==3.10.18
+overrides==7.7.0
+packaging==24.2
+pandas==2.2.3
+parso==0.8.4
+patsy==1.0.1
+pexpect==4.9.0
+pillow==11.1.0
+platformdirs==4.3.7
+posthog==4.0.1
+primp==0.14.0
+prompt_toolkit==3.0.50
+propcache==0.3.1
+protobuf==5.29.4
+psutil==7.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+py_rust_stemmers==0.1.5
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pydantic==2.11.3
+pydantic-settings==2.8.1
+pydantic_core==2.33.1
+Pygments==2.19.1
+pyparsing==3.2.3
+PyPika==0.48.9
+pyproject_hooks==1.2.0
+pyreadline3==3.5.4
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.0
+python-multipart==0.0.20
+pytz==2025.2
+pywin32==310
+PyYAML==6.0.2
+pyzmq==26.4.0
+RapidFuzz==3.13.0
+redis==5.2.1
+redis-cli==1.0.1
+referencing==0.36.2
+regex==2024.11.6
+requests==2.32.3
+requests-oauthlib==2.0.0
+rich==14.0.0
+rich-toolkit==0.14.1
+rpds-py==0.24.0
+rsa==4.9.1
+safetensors==0.5.3
+scikit-learn==1.6.1
+scipy==1.15.2
+seaborn==0.13.2
+shellingham==1.5.4
+six==1.17.0
+smmap==5.0.2
+sniffio==1.3.1
+SQLAlchemy==2.0.40
+stack-data==0.6.3
+starlette==0.45.3
+statsmodels==0.14.4
+sympy==1.13.3
+tabulate==0.9.0
+tenacity==9.1.2
+threadpoolctl==3.6.0
+tiktoken==0.9.0
+tokenizers==0.21.1
+tomli==2.2.1
+torch==2.7.0
+tornado==6.4.2
+tqdm==4.67.1
+traitlets==5.14.3
+transformers==4.51.3
+typer==0.15.2
+typing-inspect==0.9.0
+typing-inspection==0.4.0
+typing_extensions==4.13.2
+tzdata==2025.2
+urllib3==2.4.0
+uv==0.6.14
+uvicorn==0.34.0
+watchfiles==1.0.5
+wcwidth==0.2.13
+websocket-client==1.8.0
+websockets==15.0.1
+win32_setctime==1.2.0
+wrapt==1.17.2
+yarl==1.20.0
+zipp==3.21.0

src/api/__init__.py

@@ -0,0 +1,5 @@
+from fastapi import APIRouter
+from .v1 import v1_router
+
+router = APIRouter()
+router.include_router(v1_router, prefix='/v1')

src/api/v1/__init__.py

@@ -0,0 +1,9 @@
+from fastapi import APIRouter
+from .build_ml_plan import build_ml_plan_router
+from .eda_engine import eda_engine_router
+
+v1_router = APIRouter()
+
+v1_router.include_router(build_ml_plan_router, prefix='/build-ml-plan')
+v1_router.include_router(eda_engine_router, prefix='/eda-engine')
+

src/api/v1/build_ml_plan/__init__.py

@@ -0,0 +1,7 @@
+from fastapi import APIRouter
+from .task_analysis import analysis_router
+from .eda import eda_router
+
+build_ml_plan_router = APIRouter()
+build_ml_plan_router.include_router(analysis_router, prefix="/analyze-task")
+# build_ml_plan_router.include_router(eda_router, prefix="/run-eda")

src/api/v1/build_ml_plan/eda.py

@@ -0,0 +1,25 @@
+from fastapi import APIRouter
+from src.app.schemas.requests.eda import EdaRequestSchema
+from src.app.pipelines.eda.pipeline import EdaLoop
+import os
+
+eda_router = APIRouter()
+
+def delete_dir_contents(directory: str)->None:
+    for filename in os.listdir(directory):
+        file_path = os.path.join(directory, filename)
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+
+@eda_router.post('/')
+async def main(response: EdaRequestSchema):
+
+    eda = EdaLoop(payload=response, verbose=True)
+    logs = eda.loop(verbose=True)
+
+    delete_dir_contents(directory="src/core/cache/dataset_logs")
+    delete_dir_contents(directory="src/core/cache/downloads")
+
+    return {
+        "execution_logs": logs
+    }

src/api/v1/build_ml_plan/task_analysis.py

@@ -0,0 +1,54 @@
+from fastapi import APIRouter, File, UploadFile, Form, HTTPException, Request
+from src.app.pipelines.task_analysis import MLImplementationPlannerWorkflow, MLAnalysisWorkflow
+from src.app.schemas.requests.eda import EdaRequestSchema
+from src.core.utils import logger
+from typing import Optional
+import os
+import shutil
+
+analysis_router = APIRouter()
+
+UPLOAD_DIR = "src/core/cache/downloads"
+
+@analysis_router.post("/")
+async def main(
+    request: Request,
+    user_input: str = Form(...),
+    file: UploadFile = File(...)
+):
+    '''Retrieving config from app state. This config is also stored in redis cache'''
+    config = request.app.state.config
+
+    if not file.filename.endswith('.csv'):
+        logger.error("Only CSV files are allowed", log_type="api: /analyze-task", console=config['app']['verbose'])
+        raise HTTPException(status_code=400, detail="Only CSV files are allowed.")
+
+    file_path = os.path.join(UPLOAD_DIR, file.filename)
+
+    '''Storing the file in /downloads'''
+    try:
+        with open(file_path, "wb") as buffer:
+            shutil.copyfileobj(file.file, buffer)
+        logger.info("File uploaded successfully!", log_type="api: /analyze-task", console=config['app']['verbose'])
+    except Exception as e:
+        logger.error(f"Error saving file: {str(e)}", log_type="api: /analyze-task", console=config['app']['verbose'])
+        raise HTTPException(status_code=500, detail=f"Error saving file: {str(e)}")
+    
+
+    ml_analysis_wf = MLAnalysisWorkflow(user_prompt=user_input)
+    ml_analysis_results = ml_analysis_wf.run(verbose=True)
+
+    ml_imp_planner_wf = MLImplementationPlannerWorkflow(requirements_analysis=ml_analysis_results[0], technical_research=ml_analysis_results[1])
+    ml_imp_planner_results = ml_imp_planner_wf.run(verbose=True)
+
+    api_response = EdaRequestSchema(
+        dataset_path = file_path,
+        user_prompt = user_input,
+        requirement_analysis = ml_analysis_results[0],
+        technical_research = ml_analysis_results[1],
+        implementation_plan = ml_imp_planner_results
+    )
+
+    return {
+        "response": api_response
+    }

src/api/v1/eda_engine/__init__.py

@@ -0,0 +1,12 @@
+from fastapi import APIRouter
+from .data_understanding import data_understanding_router
+from .data_statistics import data_statistics_router
+from .data_quality import data_quality_router
+from .univariate_analysis import univariate_analysis_router
+
+eda_engine_router = APIRouter()
+eda_engine_router.include_router(data_understanding_router, prefix="/get-data-understanding-context")
+eda_engine_router.include_router(data_statistics_router, prefix="/get-data-statistics")
+eda_engine_router.include_router(data_quality_router, prefix="/get-data-quality")
+eda_engine_router.include_router(univariate_analysis_router, prefix="/get-univariate-analysis")
+

src/api/v1/eda_engine/data_quality.py

@@ -0,0 +1,58 @@
+import os
+import shutil
+from fastapi import APIRouter
+from src.core.utils import logger
+from fastapi import APIRouter, UploadFile, File, HTTPException, Form
+from src.app.pipelines.modules import DataQualityAssessmentWorkflow
+
+data_quality_router = APIRouter()
+
+def delete_dir_contents(directory: str)->None:
+    for filename in os.listdir(directory):
+        file_path = os.path.join(directory, filename)
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+
+@data_quality_router.post('/')
+async def main(file: UploadFile = File(...), ml_task: str = Form(None)):
+    ''' ## This endpoint accepts a CSV file upload  to initiate the Data Quality Workflow. 
+
+        ### Parameters:
+        -----------
+        - file : CSV File for the dataset
+            
+        ### Returns:
+        --------
+        - dict: Markdown Report
+    '''
+
+    if not file.filename.endswith('.csv'):
+        raise HTTPException(status_code=400, detail="Only CSV files are allowed.")
+
+    '''Clears the /downloads folder and stores the recieved file under 'dataset.csv' '''
+    
+    downloads_path = "src/core/cache/downloads"
+    os.makedirs(downloads_path, exist_ok=True)
+    delete_dir_contents(downloads_path)
+    destination_path = os.path.join(downloads_path, "dataset.csv")
+    with open(destination_path, "wb") as buffer:
+        shutil.copyfileobj(file.file, buffer)
+
+    logger.info(f"CSV file saved to {destination_path}", log_type='eda-engine/data_quality', console=True)
+    
+    '''Runs the data quality assessment workflow'''
+
+    try:
+        ds_wf = DataQualityAssessmentWorkflow(data_source=f'{downloads_path}/dataset.csv', llm_choice="gpt-4o-mini", ml_task=ml_task)
+        results = ds_wf.run(verbose=True)
+
+        return {
+            "status": "Pipeline finished running",
+            "results": results
+        }
+
+    except Exception as e:
+        logger.error(f"DataQualityAssessmentWorkflow failed with error: {e}", log_type='eda-engine/data_quality', console=True)
+        return {
+            "status": "Pipeline failed to finish",
+        }

src/api/v1/eda_engine/data_statistics.py

@@ -0,0 +1,61 @@
+import os
+import shutil
+from fastapi import APIRouter
+from src.core.utils import logger
+from fastapi import APIRouter, UploadFile, File, HTTPException, Form
+from src.app.pipelines.modules import DataStatisticsWorkflow
+
+data_statistics_router = APIRouter()
+
+def delete_dir_contents(directory: str)->None:
+    for filename in os.listdir(directory):
+        file_path = os.path.join(directory, filename)
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+
+@data_statistics_router.post('/')
+async def main(file: UploadFile = File(...),  ml_task: str = Form(None)):
+    ''' ## This endpoint accepts a CSV file upload  to initiate the Data Statistics Workflow. 
+
+        ### Parameters:
+        -----------
+        - file : CSV File for the dataset
+        \n
+        - ml_task : Final machine learning task/target
+            
+        ### Returns:
+        --------
+        - dict: Markdown Report
+    '''
+
+    if not file.filename.endswith('.csv'):
+        raise HTTPException(status_code=400, detail="Only CSV files are allowed.")
+
+    '''Clears the /downloads folder and stores the recieved file under 'dataset.csv' '''
+    
+    downloads_path = "src/core/cache/downloads"
+    os.makedirs(downloads_path, exist_ok=True)
+    delete_dir_contents(downloads_path)
+    destination_path = os.path.join(downloads_path, "dataset.csv")
+    with open(destination_path, "wb") as buffer:
+        shutil.copyfileobj(file.file, buffer)
+
+    logger.info(f"CSV file saved to {destination_path}", log_type='eda-engine/data_statistics', console=True)
+    
+    '''Runs the data statistics workflow'''
+    try:
+        ds_wf = DataStatisticsWorkflow(data_source=f'{downloads_path}/dataset.csv', llm_choice="gpt-4o-mini", ml_task=ml_task)
+        results = ds_wf.run(verbose=True)
+
+        return {
+            "status": "Pipeline finished running",
+            "results": results
+        }
+
+    except Exception as e:
+        logger.error(f"DataStatisticsWorkflow failed with error: {e}", log_type='eda-engine/data_statistics', console=True)
+        return {
+            "status": "Pipeline failed to finish",
+        }
+
+

src/api/v1/eda_engine/data_understanding.py

@@ -0,0 +1,61 @@
+import os
+import shutil
+from typing import Optional
+from src.core.utils import logger
+from fastapi import APIRouter, UploadFile, File, HTTPException, Form
+from src.app.pipelines.modules import DataUnderstandingContextWorkflow
+
+data_understanding_router = APIRouter()
+
+def delete_dir_contents(directory: str) -> None:
+    for filename in os.listdir(directory):
+        file_path = os.path.join(directory, filename)
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+
+@data_understanding_router.post('/')
+async def main(file: UploadFile = File(...), business_requirements: Optional[str] = Form(None)):
+    ''' ## This endpoint accepts a CSV file upload & additional business requirements/context to initiate the Data Understanding Context Workflow. 
+
+        ### Parameters:
+        -----------
+        - file : CSV File for the dataset
+        \n    
+        - business_context : Additional business context information about the dataset
+            
+        ### Returns:
+        --------
+        - dict: Markdown Report
+    '''
+
+    if not file.filename.endswith('.csv'):
+        raise HTTPException(status_code=400, detail="Only CSV files are allowed.")
+
+    '''Clears the /downloads folder and stores the recieved file under 'dataset.csv' '''
+    
+    downloads_path = "src/core/cache/downloads"
+    os.makedirs(downloads_path, exist_ok=True)
+    delete_dir_contents(downloads_path)
+    destination_path = os.path.join(downloads_path, "dataset.csv")
+    with open(destination_path, "wb") as buffer:
+        shutil.copyfileobj(file.file, buffer)
+
+    logger.info(f"CSV file saved to {destination_path}", log_type='eda-engine/data_understanding', console=True)
+
+
+    '''Runs the data understanding workflow'''
+
+    try:
+        duc_wf = DataUnderstandingContextWorkflow(data_source=f'{downloads_path}/dataset.csv', llm_choice="gpt-4o-mini", business_context=business_requirements)
+        results = duc_wf.run(verbose=True)
+
+        return {
+            "status": "Pipeline finished running",
+            "results": results
+        }
+
+    except Exception as e:
+        logger.error(f"DataUnderstandingContextWorkflow failed with error: {e}", log_type='eda-engine/data_understanding', console=True)
+        return {
+            "status": "Pipeline failed to finish",
+        }

src/api/v1/eda_engine/univariate_analysis.py

@@ -0,0 +1,59 @@
+import os
+import shutil
+from typing import Optional
+from src.core.utils import logger
+from fastapi import APIRouter, UploadFile, File, HTTPException, Form
+from src.app.pipelines.modules import UnivariateAnalysisWorkflow
+
+univariate_analysis_router = APIRouter()
+
+def delete_dir_contents(directory: str)->None:
+    for filename in os.listdir(directory):
+        file_path = os.path.join(directory, filename)
+        if os.path.isfile(file_path):
+            os.remove(file_path)
+
+@univariate_analysis_router.post('/')
+async def main(file: UploadFile = File(...), ml_task: str = Form(None)):
+    ''' ## This endpoint accepts a CSV file upload  to initiate the Univarite Analysis Workflow. 
+
+        ### Parameters:
+        -----------
+        - file : CSV File for the dataset
+        \n    
+        - ml_task : Final machine learning task/target
+
+        ### Returns:
+        --------
+        - dict: Markdown Report
+    '''
+
+    if not file.filename.endswith('.csv'):
+        raise HTTPException(status_code=400, detail="Only CSV files are allowed.")
+
+    '''Clears the /downloads folder and stores the recieved file under 'dataset.csv' '''
+    
+    downloads_path = "src/core/cache/downloads"
+    os.makedirs(downloads_path, exist_ok=True)
+    delete_dir_contents(downloads_path)
+    destination_path = os.path.join(downloads_path, "dataset.csv")
+    with open(destination_path, "wb") as buffer:
+        shutil.copyfileobj(file.file, buffer)
+
+    logger.info(f"CSV file saved to {destination_path}", log_type='eda-engine/univariate_analysis', console=True)
+    
+    '''Runs the data univariate analysis workflow'''
+    try:
+        ua_wf = UnivariateAnalysisWorkflow(data_source=f'{downloads_path}/dataset.csv', llm_choice="gpt-4o-mini", ml_task=ml_task)
+        results = ua_wf.run(verbose=True)
+
+        return {
+            "status": "Pipeline finished running",
+            "results": results
+        }
+
+    except Exception as e:
+        logger.error(f"UnivariateAnalysisWorkflow failed with error: {e}", log_type='eda-engine/dataunivariate_analysis_statistics', console=True)
+        return {
+            "status": "Pipeline failed to finish"
+        }

src/app/pipelines/eda/agents/agents.py

@@ -0,0 +1,65 @@
+from .prompts import intel_agent_desc, intel_agent_instructions, orchestrator_agent_desc, orchestrator_agent_instructions, analyzer_agent_desc, analyzer_agent_instructions, judging_agent_desc, judging_agent_instructions
+from .models import IntelAgentResponseSchema, OrchestratorAgentResponseSchema, AnalyzerAgentResponseSchema, JudgingAgentResponseSchema
+from agno.models.openai import OpenAIChat  # type: ignore
+from agno.agent import Agent  # type: ignore
+from dotenv import load_dotenv
+from typing import List
+import os
+
+load_dotenv()
+
+class AgentClass:
+    def __init__(self):
+        self.llm = OpenAIChat(id="gpt-4o-mini", api_key=os.getenv("OPENAI_API_KEY"))
+        self.agents = {}
+
+        self.build_agent(
+            agent_name = "intel_agent",
+            agent_desc = intel_agent_desc,
+            agent_instructions = intel_agent_instructions,
+            agent_response_model = IntelAgentResponseSchema,
+            tools = None
+        )
+
+        self.build_agent(
+            agent_name = "orchestrator_agent",
+            agent_desc = orchestrator_agent_desc,
+            agent_instructions = orchestrator_agent_instructions,
+            agent_response_model = OrchestratorAgentResponseSchema,
+            tools = None
+        )
+
+        self.build_agent(
+            agent_name = "analyzer_agent",
+            agent_desc = analyzer_agent_desc,
+            agent_instructions = analyzer_agent_instructions,
+            agent_response_model = AnalyzerAgentResponseSchema,
+            tools = None
+        )
+
+        self.build_agent(
+            agent_name = "judging_agent",
+            agent_desc = judging_agent_desc,
+            agent_instructions = judging_agent_instructions,
+            agent_response_model = JudgingAgentResponseSchema,
+            tools = None
+        )
+
+
+    def build_agent(self, agent_name: str, agent_desc: str, agent_instructions: List[str], agent_response_model, tools=None, debug_mode=False):
+        '''
+        Builds or re-builds an agent dynamically
+        '''
+        self.agents[agent_name] = Agent(
+            model = self.llm,
+            description = agent_desc,
+            instructions = agent_instructions,
+            response_model = agent_response_model,
+            structured_outputs = True,
+            tools = tools,
+            debug_mode=debug_mode,
+        )
+
+    
+
+

src/app/pipelines/eda/agents/models.py

@@ -0,0 +1,77 @@
+from pydantic import Field, BaseModel
+from ..model import DatasetSummary
+from typing import Optional
+
+class IntelAgentResponseSchema(BaseModel):
+    dataset_description: str = Field(
+        default=None,
+        description = "Gives an idea about the dataset while also containing insights from the business understanding and task goal"
+    )
+    ml_task: str = Field(
+        default=None,
+        description = "Contains the task type which will help in the EDA process"
+    )
+    dataset_summary: DatasetSummary = Field(
+        default=None,
+        description = "Contains the summary of the dataset including num of rows, columns, features, target variables"
+    )
+
+class OrchestratorAgentResponseSchema(BaseModel):
+    tool_name: str = Field(
+        default=None,
+        description="Name of the tool selected for execution"
+    )
+    justification: str = Field(
+        default=None,
+        description="Reason for selecting the tool"
+    )
+
+class AnalyzerAgentResponseSchema(BaseModel):
+    key_insights: str = Field(
+        default=None,
+        description="Detected Patterns, Key findings, Anamolies, detected trends all must be discussed here"
+    )
+    potential_issues: str = Field(
+        default=None,
+        description="Detected Issues, Any problems found in the data, Steps that have not yet been performed, Any unresolved problems with the dataset, **Unmet Business Requirements**"
+    )
+    recommendations: str = Field(
+        default=None,
+        description="areas that still need analysis or processing"
+    )
+    final_task_achieved: bool = Field(
+        default=None,
+        description="Whether user task has been completed or not" 
+    )
+
+class ExecuterAgentResponseSchema(BaseModel):
+    executed_tool: str = Field(
+        default=None, 
+        description="Tool that was executed"
+)
+    execution_details: str = Field(
+        default=None, 
+        description="Summary of execution process"
+)
+    error_logs: Optional[str] = Field(
+        default=None, 
+        description="Error logs if execution failed"
+)
+    output_file_path: Optional[str] = Field(
+        default = None,
+        description = "Path of output file stored"
+    )
+    
+class JudgingAgentResponseSchema(BaseModel):
+    detailed_context: str = Field(
+        default=None,
+        description="Contains a detailed context about all the previously executed tools/functions"
+    )
+    stop_loop: bool = Field(
+        default=None,
+        description="Contains boolean value whether the eda loop must be stopped or not"
+    )
+    justification: str = Field(
+        default=None,
+        description="Contains the justification for its decision of stopping the loop or not"
+    )

src/app/pipelines/eda/agents/prompts.py

@@ -0,0 +1,333 @@
+intel_agent_desc = '''
+    You are a Dataset Understanding and Problem Analysis Expert.
+    Your task is to analyze the provided dataset, interpret the user's problem statement, and extract meaningful insights from both the dataset and business context.
+    Your analysis will guide the Exploratory Data Analysis (EDA) and Machine Learning (ML) pipeline setup.
+    You will identify target variables, infer feature meanings, and ensure dataset insights are actionable and relevant for model building.
+'''
+
+intel_agent_instructions = [
+    '''1. Business Understanding:
+        - Analyze the user's goal from the user's prompt and business requirements to determine the nature of the problem.
+        - Identify the type of ML task (e.g., classification, regression, clustering, ranking, recommendation, etc.).
+        - Determine relevant target variable(s) based on the problem statement and dataset structure.
+        - For numerical features, interpret the statistics like mean, min., max. and std value and use this analysis to generate informative feature descriptions
+    ''',
+    '''2. Dataset Analysis:
+        - Extract key dataset metadata, including the number of rows, columns, and data types.
+        - Identify the target variable(s) that align with the task goal.
+        - Examine a **random sample of 10 rows** to infer initial insights about categorical and numerical features.
+        - For numerical features, interpret the statistics like mean, min., max. and std value and use this analysis to generate informative feature descriptions
+    ''',
+    '''3. Output Generation:
+        - Construct a structured dataset summary with:
+            - Feature descriptions, feature types (numerical, categorical, text), and missing value counts.
+            - Key statistics for numerical features (mean, min, max, std deviation) with interpretations.
+        - Define a clear **ML task type** that informs feature selection and EDA.
+        - Provide a refined list of **target variables** aligned with the business problem.
+        - Ensure that dataset insights are **actionable** and relevant for feature engineering and model training.
+        - For numerical features, interpret the statistics like mean, min., max. and std value and use this analysis to generate informative feature descriptions
+    ''',
+]
+
+orchestrator_agent_desc = '''
+      You are the Adaptive Data Analysis & Processing Decision-Making Agent. You are an expert at dynamically selecting and sequencing the most relevant data analysis, preprocessing, and transformation steps based on user instructions, dataset characteristics, and analysis reports from previously executed functions.
+      Your primary responsibility is to create a customized analytical pathway for each dataset, ensuring that prerequisite tasks are completed in an optimal sequence while avoiding unnecessary steps.
+      You operate in a continuous feedback loop, evaluating each new analysis report to refine your understanding of the dataset and adjust subsequent decisions accordingly.
+'''
+
+orchestrator_agent_instructions = [
+      '''
+      1. Understand User Intent & Business Requirements:
+          - Extract the 'Business Objective', 'ML Task', and 'Final Task' from the prompt.
+          - Identify whether the user request requires simple preprocessing, in-depth analysis, feature engineering, or another transformation.
+          - Map the requested task to its position in a typical data science workflow to identify prerequisite and dependent steps.
+          - Understand the end goal to prioritize steps that directly contribute to achieving the business objective.
+      ''',
+      '''
+      2. Analyze Dataset Characteristics & Processing History:
+          - Carefully examine 'Dataset Description' and 'Dataset Overview' to identify data types, distributions, outliers, and quality issues.
+          - Review past tool executions from 'Previous Pre-processing tools executed' chronologically to build context.
+          - Critically evaluate the most recent analysis reports to understand:
+              * What issues were detected and resolved
+              * What issues remain unaddressed
+              * How the data distribution and characteristics have changed after previous transformations
+              * What new patterns or relationships have emerged from previous analyses
+          - Detect dataset-specific challenges that might require specialized handling.
+      ''',
+      '''
+      3. Make Data-Driven Decisions on Next Steps:
+          - Always start with the Univariate Analysis before executing any other analysis or pre-processing / transformation tool.
+          - Assess whether the requested operation has dependencies and determine if those dependencies have been satisfied based on previous analysis reports.
+          - Select the most appropriate tool from 'Available Tools for Execution' considering:
+              * Current dataset state as revealed by the most recent analysis
+              * Required prerequisites for the user's requested task
+              * Potential impact on downstream analyses and transformations
+          - For complex tasks like feature engineering, ensure that sufficient data understanding has been established through appropriate exploratory analyses.
+          - Adapt the analytical pathway based on insights from previous steps—if an analysis reveals unexpected patterns, reprioritize subsequent steps accordingly.
+          - Identify when a specific analysis can be skipped because previous reports have already provided the necessary information.
+      ''',
+      '''
+      4. Provide Comprehensive Justification Grounded in Analysis Reports:
+          - Reference specific findings from previous analysis reports to justify your current decision.
+          - Explain how the chosen step builds upon or addresses issues identified in previous analyses.
+          - When bypassing a seemingly logical step, cite specific evidence from previous reports showing why it's unnecessary.
+          - Connect your decision to both immediate needs and long-term business objectives.
+          - When recommending prerequisite steps, clearly articulate how they enable the user's requested task.
+      ''',
+      '''
+      5. Output Format:
+          - Your response must contain:
+            1. Step / Iteration number the entire loop is currently at
+            2. Function Name to Execute: Choose from the available tools and return the exact name of the tool. Don't add function. to the tool name. Be very precise and under no circumstance will you return a tool name out of the list.
+            3. Comprehensive Justification for Selection that references:
+                * Specific findings from previous analysis reports
+                * Current state of the dataset
+                * How this step advances toward the business objective and user's requested task
+                * Why alternative approaches were not selected
+      ''',
+      '''
+      6. Critical Rules to Follow:
+          - Never proceed with feature engineering or advanced transformations without first verifying data quality through previous analysis reports.
+          - Continuously adapt your strategy based on new insights—previous analysis results should directly influence current decisions.
+          - Balance thoroughness with efficiency—skip steps only when analysis reports provide evidence they're unnecessary.
+          - Recognize dataset-specific characteristics that require specialized treatment rather than applying a generic approach.
+          - Be opportunistic in identifying when multiple objectives can be achieved with a single analytical step.
+          - When analysis reports reveal unexpected data characteristics, be prepared to recommend a different path than originally anticipated.
+          - Always validate that a user-requested step is appropriate given the current dataset state—recommend alternatives when necessary.
+          - Consider the computational cost of each step relative to its potential value—prioritize high-impact analyses.
+      ''',
+      '''
+      7. Special Case: First Iteration Handling:
+          - If there is no history/logs of any executed tools, always start with univariate analysis.
+          - If the user has requested a very specific function, check if it requires any prerequisite steps.
+          - If prerequisites are needed, execute those first before running the requested function.
+          - If no specific function is requested, always execute univariate analysis as the first step.
+      ''',
+    '''
+      8. **New Rule: Ensure Each Function is Only Executed Once:**
+          - If a function or type of analysis has already been executed, **do not execute it again** under any circumstance.
+          - Keep track of all previously executed functions using the history/logs.
+          - If a user requests a function that has already been run, return a justification explaining why it will not be repeated.
+          - Instead of re-running a function, suggest alternative actions based on current dataset needs and past analysis results.
+      '''
+]
+
+analyzer_agent_desc = """
+    You are the Analysis Interpretation Agent.
+    You are an expert at extracting meaningful insights from analysis results, identifying issues, and determining logical next steps in a data processing workflow.
+    Your task is to interpret the results of previously executed functions, provide detailed insights, and suggest relevant next steps based on the dataset characteristics, business requirements, and detected issues.
+    Your output will serve as input for a decision-making agent that determines which function to execute next.
+    Additionally, you must determine whether the user's task has been successfully completed based on the business objective and user objective.
+"""
+
+analyzer_agent_instructions = [
+    """
+    1. Understand the Received Inputs:
+        - Analyze the 'Function Execution Details' (a short summary of results) to grasp key takeaways.
+        - Review 'Function Metadata' to understand what steps were taken to generate the results.
+        - Carefully examine the full structured output of the function execution to extract insights.
+        - Understand the 'Business Objective' and 'User Objective' to assess completeness.
+    """,
+    """
+    2. Analyze & Interpret Function Results
+        - Examine the structure of the function output.
+        - Determine what insights are meaningful based on the type of analysis performed.
+        - Adaptively interpret key takeaways without assuming a fixed output schema.
+    """,
+    """
+    3. Identify Issues & Challenges
+        - If the function output provides statistical information, look for:
+            * Patterns, anomalies, or inconsistencies.
+        - If it generates transformed data, check:
+            * Completeness, correctness, and adherence to expected formats.
+        - If it's a model training result, extract:
+            * Performance metrics, overfitting risks, and areas for improvement.
+        - For any other type of output, assess:
+            * How well the function achieved its intended goal.
+    """,
+    """
+    4. Determine Logical Next Steps
+        - Based on the extracted insights, suggest what should logically follow:
+            * Further data cleaning or transformation?
+            * A different type of analysis?
+        - Ensure that recommendations align with the broader business objective.
+    """,
+    """
+    5. Identify Unmet Business Requirements & Task Completion Check
+        - Check if there are remaining gaps in the analysis pipeline.
+        - Verify if the extracted insights align with the original 'Business Objective' and 'User Objective'.
+        - Determine if the previous function output sufficiently addresses the business goal.
+        - Identify  any aspects of the dataset that haven’t been sufficiently analyzed yet.
+        - If certain transformations or feature engineering steps are necessary for the final objective, highlight them.
+        - Highlight any missing steps required to complete the workflow.
+        - **Task Completion Check:**
+            - If all required steps have been performed and the results meet the business objective and user goal, return `task_completed = True`.
+            - If there are still outstanding issues, required steps, or missing insights, return `task_completed = False`.
+    """,
+    """
+    6. Output Format:
+        - Your response must contain a structured summary with the following components:
+            1. key_insights: Detected Patterns, Key findings, Anamolies, detected trends all must be discussed here.
+            2. potential_issues: Detected Issues, Any problems found in the data, Steps that have not yet been performed, Any unresolved problems with the dataset, **Unmet Business Requirements**.
+            3. recommendations: Areas that still need analysis or processing.
+            4. task_completed: Boolean (True/False) indicating if the user’s objective has been met.
+    """
+]
+
+executer_agent_desc = '''
+    You are an intelligent ML workflow execution agent responsible for executing selected tools on the dataset and generating a detailed execution report. Your primary role is to apply the given tool to the dataset, monitor the process, and log the results.
+
+    At each iteration, you must:
+        1. Receive execution parameters, including the tool name, expected changes, and dataset description.
+        2. Execute the specified tool with appropriate parameters.
+        3. Capture the execution outcome, including success status, modifications made, and any errors encountered.
+        4. Generate a structured execution report.
+        
+    Your goal is to ensure the successful execution of ML workflow steps, log any issues encountered, and maintain detailed execution records.
+'''
+
+executer_agent_instructions = [
+    '''1. Input Processing
+            Upon receiving an execution request, analyze the following inputs:
+
+            - Step Number: The iteration number in the pipeline.
+            - Tool Name: The tool to be executed.
+            - Expected Changes: Modifications expected after execution.
+            - Dataset Description: A textual overview of dataset characteristics.
+            - Dataset Overview: Key statistics, structure, feature types, and known issues.
+
+            Validate whether the provided tool is applicable based on the dataset state.
+    ''',
+    '''2. Tool Execution
+            Execute the specified tool using the appropriate method and parameters. Ensure:
+
+            - Correct application of the tool based on dataset properties.
+            - Efficient execution without unnecessary operations.
+            - Handling of potential issues such as missing data, outliers, or incompatible transformations.
+
+            If execution fails, capture detailed error logs for debugging.
+    ''',
+    '''3. Capture Execution Outcome
+            After execution, document the results:
+
+            - execution_successful: Whether the tool was executed without errors.
+            - execution_details: Summary of operations performed by the tool.
+            In this execution_details analyze the python code and summarize whats happening in the code. Don't be vague be very precise. For eg. Detected 12 outliers and removed them etc
+            - error_logs: If execution failed, log relevant error messages.
+
+            Ensure all outputs are structured and informative.
+    ''',
+    '''4. Generate Execution Report
+            Construct a structured execution report with:
+
+            - step_number: Iteration number.
+            - executed_tool: Name of the executed tool.
+            - execution_successful: Boolean flag indicating success or failure.
+            - execution_details: Summary of what was done.
+            - error_logs: Error messages, if any.
+
+            This report must be clear, precise, and provide actionable insights for further processing.
+    ''',
+    '''5. You must also output the path of file stored after executing the function. You must output the exact path where the file is stored after executing the function
+          Map this to field: 'output_file_path'  
+
+          If there is no output path provided explicitly then return None
+
+    '''
+]
+
+judging_agent_desc = '''
+    You are an intelligent agent that analyzes preprocessing function execution history,
+    provides detailed context summaries, and determines when to stop machine learning loops
+    based on goal achievement and tool exhaustion analysis.
+    
+    This agent serves two primary functions:
+        1. Synthesize detailed contextual summaries of executed preprocessing functions
+        2. Make informed decisions about stopping ML loops with clear justifications
+'''
+
+judging_agent_instructions = [
+    '''1. PREPROCESSING FUNCTION ANALYSIS
+
+            The agent will receive list of previously executed preprocessing functions including:
+                - Function names
+                - Function logic descriptions
+                - Function execution results
+                
+            For each function, the agent must:
+                a) Understand the purpose and logic of the function
+                b) Analyze what the function accomplished based on its results
+                c) Track data transformations and their significance
+                d) Identify potential issues or limitations in the preprocessing steps
+                e) Recognize how each function contributes to the overall ML pipeline
+                
+            The agent must then generate a comprehensive paragraph that:
+                - Chronologically describes the preprocessing journey
+                - Highlights key transformations and their significance
+                - Identifies data quality improvements
+                - Notes important statistical properties revealed
+                - Explains how each preprocessing step prepares data for modeling
+                - Uses technical but accessible language
+                - Provides quantitative details where relevant
+                
+        2. ML LOOP TERMINATION DECISION
+            The agent will receive:
+                - The user's stated ML goal/task
+                - A list of available tools/functions in the system (provided explicitly)
+                - The history of previously executed tools/functions
+                
+            The agent must decide whether to stop the ML loop by analyzing only the provided list and history, without speculating about unlisted tools:
+                a) Goal Achievement Analysis:
+                    - Compare current state to user's goal
+                    - Check if performance metrics meet or exceed success criteria
+                    - Confirm if the primary objective has been satisfied
+                    - Determine if further iterations yield meaningful improvements
+                    
+                b) Tool Exhaustion Analysis:
+                    - Identify which available tools from the provided list have been executed
+                    - Determine which remaining tools (if any) from the provided list are still unused
+                    - If no unused tools remain, the loop must terminate
+                    
+                c) Diminishing Returns Analysis:
+                    - Review recent iteration metrics for signs of plateau or minimal gains
+                    - Evaluate cost-effectiveness of additional iterations
+
+            Decision Rules (based strictly on the provided tools list):
+                - STOP the loop (stop_loop = True) if ANY of the following conditions are met:
+                    1. The user's goal has been achieved with satisfactory results
+                    2. All tools in the provided list have been executed
+                    3. Remaining provided tools (if any) cannot meaningfully improve results toward the goal
+                    4. Recent iterations show diminishing returns below a meaningful threshold
+                - CONTINUE the loop (stop_loop = False) if ALL of the following conditions are met:
+                    1. The user's goal is not yet achieved
+                    2. There are unused tools in the provided list
+                    3. There is clear evidence that applying an unused tool could improve results
+
+            Additional Constraints:
+                - Each available tool may be executed at most once in the ML loop
+                - The agent must not refer to hypothetical or non-existent tools
+                - Decisions must be based exclusively on the provided list of tools
+
+            Justification Requirements:
+                For any decision, provide a detailed justification that:
+                    - References specific evidence from the execution history
+                    - Cites relevant performance metrics and their values
+                    - Explains the reasoning process step-by-step
+                    - Acknowledges any limitations or uncertainties
+                    - Connects the decision directly to the user's stated goal
+                    - If continuing, recommends which unused tool to apply next and why
+
+        3. OUTPUT FORMATTING
+            The agent must return a JSON object matching JudgeAgentResponseSchema with the following keys:
+
+                a) detailed_context: A comprehensive paragraph describing all executed preprocessing steps
+                and their results, explaining the ML pipeline's current state.
+
+                b) stop_loop: A boolean value (True or False) indicating whether the ML loop should stop.
+
+                c) justification: A detailed explanation of the decision to stop or continue the loop,
+                based on goal achievement, tool exhaustion, and diminishing returns analyses.
+
+        NOTE: If no additional tools from the provided list remain that can further the user's goal, the agent must set stop_loop to True.
+    '''
+]

src/app/pipelines/eda/helper.py

@@ -0,0 +1,89 @@
+import pandas as pd
+from typing import Union
+from .model import DatasetSummary, FeatureStatistics, Feature
+from src.core.utils import logger
+
+def get_summary(dataframe: pd.DataFrame)->DatasetSummary:
+    """
+    Generates a summary of the dataset, including feature details such as type, missing values,
+    and basic statistics for numerical columns.
+    
+    Args:
+        dataframe (pd.DataFrame): The input dataset as a Pandas DataFrame.
+    
+    Returns:
+        DatasetSummary: A structured summary of the dataset.
+    """
+
+    feature_list = []
+
+    for col in dataframe.columns:
+        column_name = col
+        dtype = dataframe[col].dtype
+        
+        if pd.api.types.is_numeric_dtype(dataframe[col]):
+            category = "Numerical"
+            statistics = FeatureStatistics(
+                mean = dataframe[col].mean(),
+                min_value = dataframe[col].min(),
+                max_value = dataframe[col].max(),
+                standard_deviation_value = dataframe[col].std()
+            )
+        else:
+            category = "Non-Numerical"
+            statistics = None
+        
+        null_count = dataframe[col].isnull().sum()
+        unique_count = dataframe[col].nunique()
+
+        feature = Feature(
+            feature_name = column_name,
+            feature_description = None,
+            feature_category = category,
+            feature_data_type = str(dtype),
+            missing_values = null_count,
+            unique_values = unique_count,
+            statistics = statistics
+        )
+
+        feature_list.append(feature)
+
+    return DatasetSummary(
+        num_rows = dataframe.shape[0],
+        num_features = dataframe.shape[1],
+        features = feature_list,
+        target_features = None
+    )
+
+def sample_dataset(dataset: pd.DataFrame):
+    '''Sample 10 random values from the dataset for prompts'''
+    try:
+        sampled_data = dataset.sample(n=10, random_state=42).to_string(index=False)
+        return sampled_data
+    except ValueError:
+        logger.error("Failed to sample from the dataset", log_type="eda", console=True)
+        return None
+
+def get_feature_summary(data: DatasetSummary):
+    '''Prepares a string which includes all features with their respective details'''
+    feature_details = []
+
+    for index, feature in enumerate(data.features):
+        stats = (
+            f"Mean: {feature.statistics.mean}, Min: {feature.statistics.min_value}, "
+            f"Max: {feature.statistics.max_value}, Std Dev: {feature.statistics.standard_deviation_value}"
+            if feature.statistics else "No statistics available"
+        )
+
+        feature_details.append(f"""
+        {index + 1}. '{feature.feature_name}'
+            - Description: {feature.feature_description}
+            - Category: {feature.feature_category}
+            - Data type: {feature.feature_data_type}
+            - Unique Values: {feature.unique_values}
+            - Missing Values: {feature.missing_values}
+            - Statistics: {stats}
+        """)
+
+    feature_summary = "\n".join(feature_details)
+    return feature_summary

src/app/pipelines/eda/model.py

@@ -0,0 +1,78 @@
+from pydantic import Field, BaseModel
+from typing import Optional, List
+
+class FeatureStatistics(BaseModel):
+    """
+    Provides the statistics of numerical features (columns) in the dataset.
+    """
+    mean: float = Field(
+        default=None,
+        description = "The mean value of this feature"        
+    )
+    min_value: float = Field(
+        default=None,
+        description = "The minimum value of this feature"        
+    )
+    max_value: float = Field(
+        default=None,
+        description = "The maximum value of this feature"        
+    )
+    standard_deviation_value: float = Field(
+        default=None,
+        description = "The standard deviation of this feature"        
+    )
+
+class Feature(BaseModel):
+    """
+    Represents a feature (column) in the dataset.
+    """
+    feature_name: str = Field(
+        default=None,
+        description = "The name of the feature/column"
+    )
+    feature_description: Optional[str] = Field(
+        default=None,
+        description = "A short description of the feature/column"
+    )
+    feature_category: str = Field(
+        default=None,
+        description= "Whether a feature is numerical or textual"        
+    )
+    feature_data_type: str = Field(
+        default=None,
+        description= "The data type of the feature/column"
+    )
+    unique_values: int = Field(
+        default=None,
+        description= "The number of unique values in this particular column"     
+    )
+    missing_values: int = Field(
+        default=None,
+        description= "The number of missing values in this particular column"        
+    )
+    statistics: Optional[FeatureStatistics] = Field(
+        default=None,
+        description= "Provides statistics for numerical feature"        
+    )
+
+class DatasetSummary(BaseModel):
+    """
+    Provides a summary of the dataset, including its structure and target features.
+    """
+    num_rows: int = Field(
+        default=None,
+        description = "The number of rows in the dataset"
+    )
+    num_features: int = Field(
+        default=None,
+        description = "The number of features in the dataset"
+    )
+    features: List[Feature] = Field(
+        default=None,
+        description = "A list of features"
+    )
+    target_features: Optional[List[str]] = Field(
+        default=None,
+        description = "A list of target features relevant to the business task"
+    )
+

src/app/pipelines/eda/pipeline.py

@@ -0,0 +1,256 @@
+from dotenv import load_dotenv
+from src.app.schemas.requests.eda import EdaRequestSchema
+from .helper import get_summary, get_feature_summary, sample_dataset
+from .agents.models import IntelAgentResponseSchema, ExecuterAgentResponseSchema
+from src.app.schemas.responses.eda import IterationDetails, IterationLogs
+from .agents.prompts import executer_agent_desc, executer_agent_instructions
+from .agents.agents import AgentClass
+from .tools.lib import tool_library
+from src.core.utils import logger
+from agno.agent import RunResponse # type: ignore
+from typing import Union
+import os
+import pandas as pd
+
+load_dotenv()
+
+class EdaLoop:
+    def __init__(self, payload: EdaRequestSchema, verbose=False):
+        self.ag = AgentClass()
+        self.payload = payload
+        self.dataset_overview = self._extract_details(verbose=verbose)
+
+    def _extract_details(self, verbose=False)->Union[IntelAgentResponseSchema, None]:
+        '''Prepares an initial dataset summary with feature descriptions'''
+        try:
+            logger.info("Preparing dataset summary....", log_type="eda", console=verbose)
+
+            dataset = pd.read_csv(self.payload.dataset_path)
+            dataset_summary = get_summary(dataframe=dataset)
+
+            feature_summary = get_feature_summary(data=dataset_summary)
+            sampled_data = sample_dataset(dataset=dataset)
+
+            prompt = (
+                f'User prompt: {self.payload.user_prompt}\n'
+                f'Business Understanding: {self.payload.requirement_analysis.business_understanding}\n'
+                f'Ml Task identified: {self.payload.technical_research.model_response.task}\n'
+                f'Dataset format: {self.payload.requirement_analysis.model_response.data_format}\n'
+                f'Features:\n {feature_summary}\n'
+                f'10 Sample Values from dataset:\n {sampled_data}\n'
+            )
+
+            intel_agent_response: RunResponse = self.ag.agents['intel_agent'].run(prompt, stream=False)
+            return intel_agent_response.content
+        
+        except Exception as e:
+            logger.error(f"Dataset summary preparation failed with error: {e}", log_type="eda", console=verbose)
+            return None
+    
+    def _build_orchestrator_prompt(self, execution_logs=None)->str:
+        available_tools_str = ""
+        for idx, tool in enumerate(tool_library):
+            idx+=1
+            tool_data = f"{idx}. Tool name: {tool}\n   Tool details: {tool_library[tool]['metadata']}\n"
+            available_tools_str += tool_data
+
+        execution_history = ""
+        if execution_logs:
+            execution_history = '\n'.join(execution_logs)
+        
+        prompt = (
+            f'User Prompt: {self.payload.user_prompt}\n'
+            f'Task: {self.dataset_overview.ml_task}\n'
+            f'Business Understanding: {self.payload.requirement_analysis.business_understanding}\n'
+            f'Requirements: {self.payload.requirement_analysis.model_response.technical_requirements}\n' 
+            f'Constraints: {self.payload.requirement_analysis.model_response.constraints}\n\n'
+            # f'Previous Pre-Processing Steps Done: \n{execution_history}\n\n'
+            f'Detailed Execution Summary: \n{self.execution_summary}\n\n'
+            f'Tool Library: \n{available_tools_str}\n'
+        )
+
+        return prompt
+    
+    def _build_judging_prompt(self, execution_logs)->str:
+        execution_history = '\n'.join(execution_logs)
+
+        available_tools_str = ""
+        for idx, tool in enumerate(tool_library):
+            idx+=1
+            tool_data = f"{idx}. Tool name: {tool}\n"
+            available_tools_str += tool_data
+
+        prompt = (
+            f'User Task:\n {self.payload.user_prompt}\n'
+            f'Businesss Understanding / Goals:\n {self.payload.requirement_analysis.business_understanding}\n'
+            f'Execution History:\n {execution_history}'
+            f'Available Tools:\n {available_tools_str}\n'
+        )
+
+        return prompt
+
+    def _build_analyzer_prompt(self, llm_response, tool_executed)->str:
+        prompt = (
+            f"Name of the function executed: {tool_executed['name']}\n"
+            f"Function Details: {tool_executed['metadata']}"
+            f'Function execution details:\n{llm_response.content.execution_details}\n'
+            f'Function results:\n{llm_response.messages[3].content}'
+        )
+        return prompt
+
+    def loop(self, verbose=False)->IterationLogs:
+
+        logger.info(f"Starting EDA loop with available tools: {tool_library.keys()}", log_type="eda", console=verbose)
+
+        execution_logs = []
+        iteration_logs = []
+        self.execution_summary = None
+        iteration_count = 1
+
+        recent_data_stored_path = self.payload.dataset_path
+
+        while(True):
+            
+            logger.info(f"Running Iteration {iteration_count}", log_type="eda", console=verbose)
+
+            '''==================== ORCHESTRATOR SEGMENT ===================='''
+
+            orchestrator_prompt = self._build_orchestrator_prompt(execution_logs=execution_logs)
+            
+            try:
+                orchestrator_agent_response: RunResponse = self.ag.agents['orchestrator_agent'].run(orchestrator_prompt)
+            except Exception as e:
+                logger.error(f"Failed to generate response from orchestator with error: {e}", log_type='eda', console=verbose)
+                continue
+
+            if isinstance(orchestrator_agent_response.content, str):
+                logger.error("Failed to fit orchestrator response to data model", log_type='eda', console=verbose)
+                continue
+            
+            try:
+                selected_tool_for_executioner = tool_library[orchestrator_agent_response.content.tool_name]
+            except Exception as e:
+                logger.error(f"Tool allocation failed with error: {e}", log_type='eda', console=verbose)
+                continue
+                
+            logger.info(f"Executing tool: {orchestrator_agent_response.content.tool_name}. Justification: {orchestrator_agent_response.content.justification}", log_type="eda", console=verbose)
+
+            '''==================== TOOL EXECUTION SEGMENT ===================='''
+            
+            target_features = ""
+            if self.dataset_overview.dataset_summary.target_features:
+                target_features = '\n'.join(self.dataset_overview.dataset_summary.target_features)
+            
+            while(True):
+                self.ag.build_agent(
+                    agent_name="executer_agent",
+                    agent_desc=executer_agent_desc,
+                    agent_instructions=executer_agent_instructions,
+                    agent_response_model=ExecuterAgentResponseSchema,
+                    tools=[selected_tool_for_executioner['function']]
+                )    
+
+                prompt = (
+                    # f"Only execute the given '{selected_tool_for_executioner}'\n"
+                    f"You can read the data from path: '{recent_data_stored_path}' to execute the given tool: '{selected_tool_for_executioner}' \n"
+                    f'Details about dataset:\n Target Features/Columns: {target_features}'
+                )
+
+                prompt = f"Read the data from path: '{recent_data_stored_path}'. Use this file path to execute the '{selected_tool_for_executioner}' tool. Target Features/Columns: {target_features}"
+                
+                try:
+                    executor_agent_response: RunResponse = self.ag.agents['executer_agent'].run(prompt, stream=False)
+                    # logger.info(f'{executor_agent_response.messages[3].content}', log_type='eda', console=verbose)
+                    _ = executor_agent_response.messages[3].content
+                    
+                    if executor_agent_response.content.output_file_path:
+                        if executor_agent_response.content.output_file_path is not None:
+                            recent_data_stored_path = executor_agent_response.content.output_file_path
+
+                    logger.info(f"Tool Executed successfully", log_type='eda', console=verbose)
+                    break
+                except Exception as e:
+                    logger.error(f"Tool execution failed with error: {e}. Trying Again....", log_type='eda', console=verbose)
+
+            '''==================== ANALYZER SEGMENT ===================='''
+
+            while(True):
+                try:
+                    logger.info("Generating analysis....", log_type='eda', console=verbose)
+
+                    analyzer_prompt = self._build_analyzer_prompt(llm_response=executor_agent_response, tool_executed=selected_tool_for_executioner)
+                    analyzer_agent_response: RunResponse = self.ag.agents['analyzer_agent'].run(analyzer_prompt, stream=False)
+
+                    if not isinstance(analyzer_agent_response.content, str):
+                        break
+
+                except Exception as e:
+                    logger.error(f"Failed to generate response from analyzer with error: {e}. Trying again.....", log_type='eda', console=verbose)
+
+            execution_details = (
+                f'Iteration / Step Number: {iteration_count}\n'
+                # f'{analyzer_prompt}\n'
+                f"Name of the function executed: {selected_tool_for_executioner['name']}\n"
+                f"Function Details: {selected_tool_for_executioner['metadata']}"
+                f'Function execution details:\n{executor_agent_response.content.execution_details}\n'
+                f'Key Details: {analyzer_agent_response.content.key_insights}\n'
+                # f'Potential Issues: {analyzer_agent_response.content.potential_issues}\n'
+                # f'Recommendations: {analyzer_agent_response.content.recommendations}\n'
+            )
+
+            # logger.info(f"Execution Details: \n{execution_details}", log_type='eda', console=verbose)
+            execution_logs.append(execution_details)
+
+            '''==================== JUDGING SEGMENT ===================='''
+
+            while(True):
+                try:
+                    logger.info("Evaluating Iteration with a Judge....", log_type='eda', console=verbose)
+
+                    juding_prompt = self._build_judging_prompt(execution_logs = execution_logs)
+                    judging_agent_response: RunResponse = self.ag.agents['judging_agent'].run(juding_prompt, stream=False)
+
+                    if not isinstance(judging_agent_response.content, str):
+                        break
+
+                except Exception as e:
+                    logger.error(f"Failed to generate response from judge with error: {e}. Trying again.....", log_type='eda', console=verbose)
+
+
+            '''==================== STORING ITERATION DETAILS SEGMENT ===================='''
+
+            iteration_logs.append(IterationDetails(
+                iteration_number = iteration_count,
+                orchestrator_response = orchestrator_agent_response.content,
+                executer_response = executor_agent_response.content,
+                analyzer_response = analyzer_agent_response.content,
+                judge_response = judging_agent_response.content
+            ))
+
+            iteration_count+=1
+
+            if judging_agent_response.content.stop_loop:
+                logger.info(f"Stop Loop = {judging_agent_response.content.stop_loop}. Justification: {judging_agent_response.content.justification}", log_type='eda', console=verbose)
+                break
+            else:
+                self.execution_summary = judging_agent_response.content.detailed_context
+                logger.info(f"Stop Loop = {judging_agent_response.content.stop_loop}.   Justification: {judging_agent_response.content.justification}", log_type='eda', console=verbose)
+
+        return IterationLogs(logs=iteration_logs)
+
+
+
+
+
+
+
+
+
+
+
+
+
+        
+        
+
+            

src/app/pipelines/eda/tools/analysis_tools/__init__.py

@@ -0,0 +1,3 @@
+from .univariate_analysis import univariate_analysis
+from .bivariate_analysis import bivariate_analysis
+from .multivariate_analysis import multivariate_analysis

src/app/pipelines/eda/tools/analysis_tools/bivariate_analysis.py

@@ -0,0 +1,1028 @@
+import pandas as pd
+import numpy as np
+from scipy import stats
+from sklearn.metrics import mutual_info_score
+from statsmodels.stats.multicomp import pairwise_tukeyhsd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import pointbiserialr
+from agno.utils.log import logger
+import json
+
+def comprehensive_bivariate_analysis(df, visualize=False, output_dir="bivariate_plots"):
+    """
+    Performs comprehensive bivariate analysis on all pairs of variables in the dataframe.
+    
+    Parameters:
+    -----------
+    df : pandas.DataFrame
+        The input dataframe to analyze
+    visualize : bool, default=True
+        Whether to generate visualizations
+    output_dir : str, default="bivariate_plots"
+        Directory to save visualization plots if visualize=True
+        
+    Returns:
+    --------
+    dict
+        A dictionary containing the results of all bivariate analyses and a summary
+    """
+    results = {
+        'numerical_vs_numerical': {},
+        'categorical_vs_categorical': {},
+        'numerical_vs_categorical': {},
+        'summary': {
+            'strongest_correlations': [],
+            'significant_category_associations': [],
+            'significant_group_differences': [],
+            'key_insights': []
+        }
+    }
+    
+    # Identify variable types
+    numerical_features = df.select_dtypes(include=['int64', 'float64']).columns.tolist()
+    categorical_features = df.select_dtypes(include=['object', 'category', 'bool']).columns.tolist()
+    
+    # Convert any low-cardinality numerical features to categorical
+    for col in numerical_features.copy():
+        if df[col].nunique() <= 10:  # Threshold for low cardinality
+            categorical_features.append(col)
+            numerical_features.remove(col)
+    
+    # 1. Numerical vs Numerical Analysis
+    if len(numerical_features) >= 2:
+        print("Starting Numerical-numerical analysis")
+        results['numerical_vs_numerical'] = analyze_numerical_numerical(df, numerical_features, visualize, output_dir)
+    
+    # 2. Categorical vs Categorical Analysis
+    if len(categorical_features) >= 2:
+        print("Starting categorical-categorical analysis")
+        results['categorical_vs_categorical'] = analyze_categorical_categorical(df, categorical_features, visualize, output_dir)
+    
+    # 3. Numerical vs Categorical Analysis
+    if len(numerical_features) >= 1 and len(categorical_features) >= 1:
+        print("Starting numerical-categorical analysis")
+        results['numerical_vs_categorical'] = analyze_numerical_categorical(df, numerical_features, categorical_features, visualize=visualize, output_dir=output_dir)
+    
+    # Generate summary of findings
+    print("Generating summaries")
+    results['summary'] = generate_summary(results)
+    
+    return results
+
+def analyze_numerical_numerical(df, numerical_features, visualize=False, output_dir="bivariate_plots"):
+    """Analyze relationships between numerical features"""
+    results = {
+        'correlations': {
+            'pearson': {},
+            'spearman': {},
+            'kendall': {}
+        },
+        'covariance': {},
+        'best_fit_relationships': {},
+        'significant_correlations': []
+    }
+    
+    # Create output directory if it doesn't exist and visualization is enabled
+    if visualize:
+        import os
+        os.makedirs(output_dir, exist_ok=True)
+    
+    # Calculate correlation matrices
+    if len(numerical_features) > 1:
+        # Pearson correlation (linear)
+        pearson_corr = df[numerical_features].corr(method='pearson')
+        # Spearman correlation (monotonic)
+        spearman_corr = df[numerical_features].corr(method='spearman')
+        # Kendall correlation (ordinal)
+        kendall_corr = df[numerical_features].corr(method='kendall')
+        
+        # Calculate p-values for Pearson correlation
+        pearson_p_values = pd.DataFrame(np.zeros_like(pearson_corr), 
+                                       index=pearson_corr.index, 
+                                       columns=pearson_corr.columns)
+        
+        # Calculate p-values for Spearman correlation
+        spearman_p_values = pd.DataFrame(np.zeros_like(spearman_corr), 
+                                       index=spearman_corr.index, 
+                                       columns=spearman_corr.columns)
+        
+        # Visualize correlation heatmaps if enabled
+        if visualize:
+            plt.figure(figsize=(12, 10))
+            sns.heatmap(pearson_corr, annot=True, cmap='coolwarm', vmin=-1, vmax=1, center=0)
+            plt.title('Pearson Correlation Heatmap')
+            plt.tight_layout()
+            plt.savefig(f"{output_dir}/pearson_correlation_heatmap.png")
+            plt.close()
+            
+            plt.figure(figsize=(12, 10))
+            sns.heatmap(spearman_corr, annot=True, cmap='coolwarm', vmin=-1, vmax=1, center=0)
+            plt.title('Spearman Correlation Heatmap')
+            plt.tight_layout()
+            plt.savefig(f"{output_dir}/spearman_correlation_heatmap.png")
+            plt.close()
+        
+        for i in range(len(numerical_features)):
+            for j in range(i+1, len(numerical_features)):
+                x = df[numerical_features[i]].values
+                y = df[numerical_features[j]].values
+                
+                # Remove NaN values
+                mask = ~(np.isnan(x) | np.isnan(y))
+                x_clean = x[mask]
+                y_clean = y[mask]
+                
+                if len(x_clean) > 2:  # Need at least 3 points for correlation
+                    # Calculate Pearson correlation and p-value
+                    r_pearson, p_pearson = stats.pearsonr(x_clean, y_clean)
+                    pearson_p_values.loc[numerical_features[i], numerical_features[j]] = p_pearson
+                    pearson_p_values.loc[numerical_features[j], numerical_features[i]] = p_pearson
+                    
+                    # Calculate Spearman correlation and p-value
+                    r_spearman, p_spearman = stats.spearmanr(x_clean, y_clean)
+                    spearman_p_values.loc[numerical_features[i], numerical_features[j]] = p_spearman
+                    spearman_p_values.loc[numerical_features[j], numerical_features[i]] = p_spearman
+                    
+                    # Analyze best fit relationship (linear, polynomial, logarithmic, exponential)
+                    if len(x_clean) >= 10:  # Only try to fit models if we have enough data
+                        best_fit_info = analyze_best_fit_relationship(x_clean, y_clean)
+                        results['best_fit_relationships'][f"{numerical_features[i]}_vs_{numerical_features[j]}"] = best_fit_info
+                    
+                    # Visualize scatterplot and regression line if enabled
+                    if visualize and abs(r_pearson) >= 0.3:  # Only plot more significant relationships
+                        plt.figure(figsize=(10, 6))
+                        sns.regplot(x=x_clean, y=y_clean, line_kws={"color":"red"})
+                        plt.xlabel(numerical_features[i])
+                        plt.ylabel(numerical_features[j])
+                        plt.title(f'{numerical_features[i]} vs {numerical_features[j]} (r={r_pearson:.3f}, p={p_pearson:.4f})')
+                        plt.savefig(f"{output_dir}/{numerical_features[i]}_vs_{numerical_features[j]}_scatter.png")
+                        plt.close()
+                    
+        # Store results
+        results['correlations']['pearson'] = {
+            'correlation_matrix': pearson_corr.to_dict(),
+            'p_values': pearson_p_values.to_dict()
+        }
+        
+        results['correlations']['spearman'] = {
+            'correlation_matrix': spearman_corr.to_dict(),
+            'p_values': spearman_p_values.to_dict()
+        }
+        
+        results['correlations']['kendall'] = {
+            'correlation_matrix': kendall_corr.to_dict()
+        }
+        
+        # Calculate covariance matrix
+        covariance_matrix = df[numerical_features].cov()
+        results['covariance'] = covariance_matrix.to_dict()
+        
+        # Find pairs with significant correlation
+        significant_pairs = []
+        for i in range(len(numerical_features)):
+            for j in range(i+1, len(numerical_features)):
+                feat_i = numerical_features[i]
+                feat_j = numerical_features[j]
+                
+                if abs(pearson_corr.loc[feat_i, feat_j]) >= 0.3:  # Lower threshold to capture more relationships
+                    p_value = pearson_p_values.loc[feat_i, feat_j]
+                    correlation_type = "positive" if pearson_corr.loc[feat_i, feat_j] > 0 else "negative"
+                    correlation_strength = ""
+                    
+                    if abs(pearson_corr.loc[feat_i, feat_j]) >= 0.8:
+                        correlation_strength = "very strong"
+                    elif abs(pearson_corr.loc[feat_i, feat_j]) >= 0.6:
+                        correlation_strength = "strong"
+                    elif abs(pearson_corr.loc[feat_i, feat_j]) >= 0.4:
+                        correlation_strength = "moderate"
+                    else:
+                        correlation_strength = "weak"
+                    
+                    # Compare with Spearman to detect non-linear relationships
+                    pearson_spearman_diff = abs(pearson_corr.loc[feat_i, feat_j] - spearman_corr.loc[feat_i, feat_j])
+                    relationship_type = "linear"
+                    if pearson_spearman_diff > 0.1:
+                        relationship_type = "potentially non-linear"
+                    
+                    significant_pairs.append({
+                        'feature_1': feat_i,
+                        'feature_2': feat_j,
+                        'pearson_correlation': pearson_corr.loc[feat_i, feat_j],
+                        'spearman_correlation': spearman_corr.loc[feat_i, feat_j],
+                        'p_value': p_value,
+                        'significant': p_value < 0.05,
+                        'correlation_type': correlation_type,
+                        'correlation_strength': correlation_strength,
+                        'relationship_type': relationship_type,
+                        'pearson_spearman_diff': pearson_spearman_diff
+                    })
+        
+        # Sort by absolute correlation value
+        significant_pairs.sort(key=lambda x: abs(x['pearson_correlation']), reverse=True)
+        results['significant_correlations'] = significant_pairs
+    
+    return results
+
+def analyze_best_fit_relationship(x, y):
+    """Analyze which type of relationship (linear, polynomial, log, exponential) best fits the data"""
+    results = {}
+    
+    # Check for non-positive values that would cause issues with log/exponential fits
+    x_min, y_min = np.min(x), np.min(y)
+    
+    # Linear fit
+    try:
+        slope, intercept, r_value, p_value, std_err = stats.linregress(x, y)
+        results['linear'] = {
+            'equation': f'y = {slope:.4f}x + {intercept:.4f}',
+            'r_squared': r_value**2,
+            'p_value': p_value
+        }
+    except:
+        results['linear'] = {'error': 'Failed to fit linear model'}
+    
+    # Polynomial fit (degree 2)
+    try:
+        coeffs = np.polyfit(x, y, 2)
+        p = np.poly1d(coeffs)
+        # Calculate R-squared
+        yhat = p(x)
+        ybar = np.mean(y)
+        ssreg = np.sum((yhat - ybar)**2)
+        sstot = np.sum((y - ybar)**2)
+        r_squared = ssreg / sstot
+        
+        results['polynomial'] = {
+            'equation': f'y = {coeffs[0]:.4f}x² + {coeffs[1]:.4f}x + {coeffs[2]:.4f}',
+            'r_squared': r_squared
+        }
+    except:
+        results['polynomial'] = {'error': 'Failed to fit polynomial model'}
+    
+    # Logarithmic fit (if x > 0)
+    if x_min > 0:
+        try:
+            coeffs = np.polyfit(np.log(x), y, 1)
+            # Calculate R-squared
+            yhat = coeffs[0] * np.log(x) + coeffs[1]
+            ybar = np.mean(y)
+            ssreg = np.sum((yhat - ybar)**2)
+            sstot = np.sum((y - ybar)**2)
+            r_squared = ssreg / sstot
+            
+            results['logarithmic'] = {
+                'equation': f'y = {coeffs[0]:.4f}ln(x) + {coeffs[1]:.4f}',
+                'r_squared': r_squared
+            }
+        except:
+            results['logarithmic'] = {'error': 'Failed to fit logarithmic model'}
+    
+    # Exponential fit (if y > 0)
+    if y_min > 0:
+        try:
+            coeffs = np.polyfit(x, np.log(y), 1)
+            # Calculate R-squared
+            yhat = np.exp(coeffs[1]) * np.exp(coeffs[0] * x)
+            ybar = np.mean(y)
+            ssreg = np.sum((yhat - ybar)**2)
+            sstot = np.sum((y - ybar)**2)
+            r_squared = ssreg / sstot
+            
+            results['exponential'] = {
+                'equation': f'y = {np.exp(coeffs[1]):.4f}e^({coeffs[0]:.4f}x)',
+                'r_squared': r_squared
+            }
+        except:
+            results['exponential'] = {'error': 'Failed to fit exponential model'}
+    
+    # Find best fit model
+    best_fit = {'model': None, 'r_squared': -1}
+    for model_type in results:
+        if 'r_squared' in results[model_type] and results[model_type]['r_squared'] > best_fit['r_squared']:
+            best_fit = {'model': model_type, 'r_squared': results[model_type]['r_squared']}
+    
+    results['best_fit'] = best_fit
+    
+    return results
+
+def analyze_categorical_categorical(df, categorical_features, visualize=False, output_dir="bivariate_plots"):
+    """Analyze relationships between categorical features"""
+    results = {
+        'chi_square_tests': {},
+        'cramers_v': {},
+        'contingency_tables': {},
+        'phi_coefficients': {},
+        'lambda_coefficients': {},
+        'significant_associations': []
+    }
+    
+    # Create output directory if it doesn't exist and visualization is enabled
+    if visualize:
+        import os
+        os.makedirs(output_dir, exist_ok=True)
+    
+    # Chi-square tests & Cramer's V
+    for i in range(len(categorical_features)):
+        for j in range(i+1, len(categorical_features)):
+            feat_i = categorical_features[i]
+            feat_j = categorical_features[j]
+            
+            # Get clean data (remove NaNs)
+            data_ij = df[[feat_i, feat_j]].dropna()
+            
+            # Skip if either feature has only one category after dropping NAs
+            if data_ij[feat_i].nunique() <= 1 or data_ij[feat_j].nunique() <= 1:
+                continue
+            
+            # Create contingency table
+            contingency_table = pd.crosstab(data_ij[feat_i], data_ij[feat_j])
+            results['contingency_tables'][f"{feat_i}_vs_{feat_j}"] = contingency_table.to_dict()
+            
+            # Create normalized contingency tables (row and column proportions)
+            row_proportions = contingency_table.div(contingency_table.sum(axis=1), axis=0)
+            col_proportions = contingency_table.div(contingency_table.sum(axis=0), axis=1)
+            
+            # Check if we have enough samples for chi-square test
+            # Rule of thumb: 80% of cells should have expected frequencies >= 5
+            chi2, p, dof, expected = stats.chi2_contingency(contingency_table)
+            expected_array = np.array(expected)
+            
+            # Calculate percentage of cells with expected frequency >= 5
+            valid_expected_percentage = np.sum(expected_array >= 5) / expected_array.size
+            
+            # Calculate Cramer's V (measure of association)
+            n = contingency_table.sum().sum()
+            min_dim = min(contingency_table.shape) - 1
+            if min_dim > 0:  # Avoid division by zero
+                cramers_v = np.sqrt(chi2 / (n * min_dim))
+            else:
+                cramers_v = np.nan
+            
+            results['chi_square_tests'][f"{feat_i}_vs_{feat_j}"] = {
+                'chi2': chi2,
+                'p_value': p,
+                'degrees_of_freedom': dof,
+                'valid_expected_percentage': valid_expected_percentage,
+                'reliable_test': valid_expected_percentage >= 0.8  # Generally accepted threshold
+            }
+            
+            results['cramers_v'][f"{feat_i}_vs_{feat_j}"] = cramers_v
+            
+            # For 2x2 contingency tables, calculate Phi coefficient
+            if contingency_table.shape == (2, 2):
+                phi_coef = np.sqrt(chi2 / n)
+                results['phi_coefficients'][f"{feat_i}_vs_{feat_j}"] = phi_coef
+            
+            # Calculate Lambda coefficient (asymmetric measure of association)
+            try:
+                # Lambda for predicting feat_j from feat_i
+                lambda_ij = calculate_lambda(contingency_table)
+                # Lambda for predicting feat_i from feat_j
+                lambda_ji = calculate_lambda(contingency_table.T)
+                
+                results['lambda_coefficients'][f"{feat_i}_vs_{feat_j}"] = {
+                    'lambda_ij': lambda_ij,  # For predicting j from i
+                    'lambda_ji': lambda_ji   # For predicting i from j
+                }
+            except:
+                pass
+            
+            # Visualize contingency table as heatmap if enabled
+            if visualize and valid_expected_percentage >= 0.8 and p < 0.05:
+                plt.figure(figsize=(10, 8))
+                sns.heatmap(contingency_table, annot=True, fmt='d', cmap='YlGnBu')
+                plt.title(f'{feat_i} vs {feat_j} (Cramer\'s V={cramers_v:.3f}, p={p:.4f})')
+                plt.savefig(f"{output_dir}/{feat_i}_vs_{feat_j}_contingency.png")
+                plt.close()
+                
+                # Also plot normalized contingency table (row proportions)
+                plt.figure(figsize=(10, 8))
+                sns.heatmap(row_proportions, annot=True, fmt='.2f', cmap='YlGnBu')
+                plt.title(f'{feat_i} vs {feat_j} (Row Proportions)')
+                plt.savefig(f"{output_dir}/{feat_i}_vs_{feat_j}_row_proportions.png")
+                plt.close()
+            
+            # Check for significant association
+            if p < 0.05 and valid_expected_percentage >= 0.8:
+                association_strength = ""
+                if cramers_v >= 0.5:
+                    association_strength = "strong"
+                elif cramers_v >= 0.3:
+                    association_strength = "moderate"
+                elif cramers_v >= 0.1:
+                    association_strength = "weak"
+                else:
+                    association_strength = "very weak"
+                
+                # Check if we calculated lambda coefficients
+                lambda_info = {}
+                if f"{feat_i}_vs_{feat_j}" in results['lambda_coefficients']:
+                    lambda_info = results['lambda_coefficients'][f"{feat_i}_vs_{feat_j}"]
+                
+                # Check if we calculated phi coefficient
+                phi_coef = None
+                if f"{feat_i}_vs_{feat_j}" in results['phi_coefficients']:
+                    phi_coef = results['phi_coefficients'][f"{feat_i}_vs_{feat_j}"]
+                
+                results['significant_associations'].append({
+                    'feature_1': feat_i,
+                    'feature_2': feat_j,
+                    'chi_square': chi2,
+                    'p_value': p,
+                    'cramers_v': cramers_v,
+                    'association_strength': association_strength,
+                    'phi_coefficient': phi_coef,
+                    'lambda_coefficients': lambda_info
+                })
+    
+    # Sort by Cramer's V
+    results['significant_associations'].sort(key=lambda x: x['cramers_v'], reverse=True)
+    
+    return results
+
+def calculate_lambda(contingency_table):
+    """
+    Calculate Goodman and Kruskal's lambda
+    Lambda measures the proportional reduction in error when values of one variable are used to predict values of another
+    """
+    # Convert to numpy array
+    cont_array = np.array(contingency_table)
+    
+    # Total observations
+    n = np.sum(cont_array)
+    
+    # Sum of maximum frequency in each row
+    row_max_sum = np.sum(np.max(cont_array, axis=1))
+    
+    # Mode of column sums (most frequent category overall)
+    col_sums = np.sum(cont_array, axis=0)
+    overall_mode = np.max(col_sums)
+    
+    # Calculate lambda
+    if overall_mode == n:
+        # No predictive ability (all observations are in one category)
+        return 0
+    else:
+        return (row_max_sum - overall_mode) / (n - overall_mode)
+
+def analyze_numerical_categorical(df, numerical_features, categorical_features, max_categories=30, sample_size=None, skip_tukey=False, visualize=False, output_dir="bivariate_plots"):
+    """
+    Optimized analysis of relationships between numerical and categorical features
+    
+    Parameters:
+    -----------
+    df : pandas.DataFrame
+        The input dataframe
+    numerical_features : list
+        List of numerical feature names
+    categorical_features : list
+        List of categorical feature names
+    max_categories : int, default=30
+        Skip categorical features with more unique values than this
+    sample_size : int or None, default=None
+        If provided, analyze a random sample of this size for large datasets
+    skip_tukey : bool, default=True
+        Skip Tukey's test which can be computationally expensive
+    visualize : bool, default=True
+        Whether to generate visualizations
+    output_dir : str, default="bivariate_plots"
+        Directory to save visualization plots if visualize=True
+    
+    Returns:
+    --------
+    dict
+        Results of numerical-categorical analysis
+    """
+    results = {
+        'anova_tests': {},
+        'effect_sizes': {},
+        'group_statistics': {},
+        'point_biserial_correlations': {},
+        'significant_differences': []
+    }
+    
+    # Create output directory if it doesn't exist and visualization is enabled
+    if visualize:
+        import os
+        os.makedirs(output_dir, exist_ok=True)
+    
+    # Performance optimization: Use a sample for large datasets
+    if sample_size is not None and len(df) > sample_size:
+        df_sample = df.sample(sample_size, random_state=42)
+    else:
+        df_sample = df
+    
+    # Pre-compute feature cardinality to avoid repeated calculations
+    cat_feature_cardinality = {feat: df_sample[feat].nunique() for feat in categorical_features}
+    
+    for num_feat in numerical_features:
+        for cat_feat in categorical_features:
+            # Skip high cardinality categorical features
+            if cat_feature_cardinality[cat_feat] > max_categories:
+                continue
+                
+            # Get clean data (remove NaNs)
+            data = df_sample[[num_feat, cat_feat]].dropna()
+            
+            # Skip if categorical feature has only one category after dropping NAs
+            if data[cat_feat].nunique() <= 1:
+                continue
+            
+            # Calculate group statistics more efficiently using agg
+            group_stats = data.groupby(cat_feat)[num_feat].agg(['count', 'mean', 'std', 'min', 'max', 'median'])
+            
+            # Calculate IQR and outlier bounds for each group
+            group_stats['q1'] = data.groupby(cat_feat)[num_feat].quantile(0.25)
+            group_stats['q3'] = data.groupby(cat_feat)[num_feat].quantile(0.75)
+            group_stats['iqr'] = group_stats['q3'] - group_stats['q1']
+            group_stats['lower_bound'] = group_stats['q1'] - 1.5 * group_stats['iqr']
+            group_stats['upper_bound'] = group_stats['q3'] + 1.5 * group_stats['iqr']
+            
+            # Convert to dictionary with simplified structure to save memory
+            results['group_statistics'][f"{num_feat}_by_{cat_feat}"] = {
+                'counts': group_stats['count'].to_dict(),
+                'means': group_stats['mean'].to_dict(),
+                'stds': group_stats['std'].to_dict(),
+                'medians': group_stats['median'].to_dict(),
+                'q1': group_stats['q1'].to_dict(),
+                'q3': group_stats['q3'].to_dict()
+            }
+            
+            # Calculate point-biserial correlation for binary categorical variables
+            if data[cat_feat].nunique() == 2:
+                try:
+                    # For binary categories, use point-biserial correlation
+                    # Need to convert categorical to numeric first
+                    cat_values = data[cat_feat].unique()
+                    binary_map = {cat_values[0]: 0, cat_values[1]: 1}
+                    data_numeric = data.copy()
+                    data_numeric[cat_feat] = data_numeric[cat_feat].map(binary_map)
+                    
+                    # Calculate point-biserial correlation
+                    corr, p_value = pointbiserialr(data_numeric[cat_feat], data_numeric[num_feat])
+                    
+                    results['point_biserial_correlations'][f"{num_feat}_by_{cat_feat}"] = {
+                        'correlation': corr,
+                        'p_value': p_value,
+                        'categorical_mapping': binary_map
+                    }
+                except Exception as e:
+                    pass
+            
+            # Get groups for ANOVA more efficiently
+            groups = [data[data[cat_feat] == cat][num_feat].values 
+                     for cat in data[cat_feat].unique() 
+                     if len(data[data[cat_feat] == cat]) > 0]
+            
+            group_labels = [cat for cat in data[cat_feat].unique() 
+                           if len(data[data[cat_feat] == cat]) > 0]
+            
+            # Create visualizations if enabled
+            if visualize:
+                # Create box plot
+                plt.figure(figsize=(12, 6))
+                sns.boxplot(x=cat_feat, y=num_feat, data=data)
+                plt.title(f'{num_feat} by {cat_feat}')
+                plt.xticks(rotation=45)
+                plt.tight_layout()
+                plt.savefig(f"{output_dir}/{num_feat}_by_{cat_feat}_boxplot.png")
+                plt.close()
+                
+                # Create violin plot for more detail on distributions
+                plt.figure(figsize=(12, 6))
+                sns.violinplot(x=cat_feat, y=num_feat, data=data)
+                plt.title(f'{num_feat} by {cat_feat} (Distribution)')
+                plt.xticks(rotation=45)
+                plt.tight_layout()
+                plt.savefig(f"{output_dir}/{num_feat}_by_{cat_feat}_violinplot.png")
+                plt.close()
+            
+            # Run ANOVA if we have at least 2 groups
+            if len(groups) >= 2:
+                try:
+                    # One-way ANOVA
+                    f_stat, p_value = stats.f_oneway(*groups)
+                    
+                    # Calculate effect size (eta-squared) more efficiently
+                    grand_mean = data[num_feat].mean()
+                    
+                    # Vectorized computation of SS between
+                    group_means = np.array([group.mean() for group in groups])
+                    group_sizes = np.array([len(group) for group in groups])
+                    ss_between = np.sum(group_sizes * (group_means - grand_mean)**2)
+                    
+                    # Vectorized computation of SS total
+                    ss_total = np.sum((data[num_feat].values - grand_mean)**2)
+                    
+                    eta_squared = ss_between / ss_total if ss_total != 0 else 0
+                    
+                    # Calculate omega-squared (less biased estimate of effect size than eta-squared)
+                    k = len(groups)  # Number of groups
+                    n = len(data)    # Total sample size
+                    df_between = k - 1
+                    df_within = n - k
+                    ms_between = ss_between / df_between if df_between > 0 else 0
+                    ss_within = ss_total - ss_between
+                    ms_within = ss_within / df_within if df_within > 0 else 0
+                    
+                    omega_squared = (ss_between - (df_between * ms_within)) / (ss_total + ms_within) if (ss_total + ms_within) != 0 else 0
+                    omega_squared = max(0, omega_squared)  # Ensure non-negative
+                    
+                    results['anova_tests'][f"{num_feat}_by_{cat_feat}"] = {
+                        'f_statistic': float(f_stat),  # Convert to native Python types to reduce memory
+                        'p_value': float(p_value),
+                        'significant': p_value < 0.05,
+                        'degrees_of_freedom_between': df_between,
+                        'degrees_of_freedom_within': df_within,
+                        'ss_between': float(ss_between),
+                        'ss_within': float(ss_within),
+                        'ss_total': float(ss_total)
+                    }
+                    
+                    results['effect_sizes'][f"{num_feat}_by_{cat_feat}"] = {
+                        'eta_squared': float(eta_squared),
+                        'omega_squared': float(omega_squared)
+                    }
+                    
+                    # Calculate Levene's test for homogeneity of variances
+                    try:
+                        levene_stat, levene_p = stats.levene(*groups)
+                        results['anova_tests'][f"{num_feat}_by_{cat_feat}"]["levene_test"] = {
+                            'statistic': float(levene_stat),
+                            'p_value': float(levene_p),
+                            'equal_variances': levene_p >= 0.05
+                        }
+                        
+                        # If variances are not equal, calculate Welch's ANOVA
+                        if levene_p < 0.05:
+                            try:
+                                from scipy.stats import f_oneway
+                                import statsmodels.api as sm
+                                from statsmodels.formula.api import ols
+                                
+                                # Create a new dataframe for Welch's test
+                                welch_data = pd.DataFrame({
+                                    'value': data[num_feat],
+                                    'group': data[cat_feat]
+                                })
+                                
+                                # Fit the model
+                                model = ols('value ~ C(group)', data=welch_data).fit()
+                                
+                                # Perform Welch's ANOVA
+                                welch_table = sm.stats.anova_lm(model, typ=2)
+                                
+                                # Extract statistics
+                                welch_f = welch_table.loc['C(group)', 'F']
+                                welch_p = welch_table.loc['C(group)', 'PR(>F)']
+                                
+                                results['anova_tests'][f"{num_feat}_by_{cat_feat}"]["welch_anova"] = {
+                                    'f_statistic': float(welch_f),
+                                    'p_value': float(welch_p),
+                                    'significant': welch_p < 0.05
+                                }
+                            except:
+                                # If Welch's ANOVA fails, skip it
+                                pass
+                    except:
+                        # If Levene's test fails, skip it
+                        pass
+                    
+                    # If ANOVA is significant, perform post-hoc Tukey's test (optional)
+                    if p_value < 0.05 and len(groups) > 2 and not skip_tukey:
+                        # Prepare data for Tukey's test
+                        all_data = np.concatenate(groups)
+                        group_labels_for_tukey = np.repeat(group_labels, [len(group) for group in groups])
+                        
+                        # Perform Tukey's test (computationally expensive)
+                        try:
+                            tukey_results = pairwise_tukeyhsd(all_data, group_labels_for_tukey)
+                            tukey_summary = pd.DataFrame(data=tukey_results._results_table.data[1:], 
+                                                       columns=tukey_results._results_table.data[0])
+                            
+                            # Store only significant pairs to save memory
+                            significant_pairs = tukey_summary[tukey_summary['p-adj'] < 0.05]
+                            significant_pairs_dict = significant_pairs.to_dict('records') if not significant_pairs.empty else []
+                            
+                            results['anova_tests'][f"{num_feat}_by_{cat_feat}"]["tukey_posthoc"] = {
+                                'significant_pairs': significant_pairs_dict
+                            }
+                            
+                            # Visualize Tukey's test results if enabled
+                            if visualize and not significant_pairs.empty:
+                                plt.figure(figsize=(12, len(significant_pairs) * 0.5 + 2))
+                                significant_pairs_plot = [(f"{row['group1']} vs {row['group2']}", 
+                                                        row['meandiff'], 
+                                                        row['lower'], 
+                                                        row['upper']) 
+                                                       for _, row in significant_pairs.iterrows()]
+                                
+                                # Sort by mean difference
+                                significant_pairs_plot.sort(key=lambda x: x[1])
+                                
+                                # Plot
+                                for i, (pair, diff, lower, upper) in enumerate(significant_pairs_plot):
+                                    plt.plot([lower, upper], [i, i], 'b-')
+                                    plt.plot([diff], [i], 'bo')
+                                
+                                plt.axvline(x=0, color='r', linestyle='--')
+                                plt.yticks(range(len(significant_pairs_plot)), [pair for pair, _, _, _ in significant_pairs_plot])
+                                plt.xlabel('Mean Difference')
+                                plt.title(f'Tukey\'s HSD: Significant Differences in {num_feat} by {cat_feat}')
+                                plt.tight_layout()
+                                plt.savefig(f"{output_dir}/{num_feat}_by_{cat_feat}_tukey.png")
+                                plt.close()
+                        except Exception as e:
+                            pass
+                    
+                    # Calculate Kruskal-Wallis test (non-parametric alternative to ANOVA)
+                    try:
+                        h_stat, kw_p_value = stats.kruskal(*groups)
+                        results['anova_tests'][f"{num_feat}_by_{cat_feat}"]["kruskal_wallis"] = {
+                            'h_statistic': float(h_stat),
+                            'p_value': float(kw_p_value),
+                            'significant': kw_p_value < 0.05
+                        }
+                        
+                        # If Kruskal-Wallis is significant, perform post-hoc Dunn's test
+                        if kw_p_value < 0.05 and len(groups) > 2 and not skip_tukey:
+                            try:
+                                from scikit_posthocs import posthoc_dunn
+                                
+                                # Create a new dataframe for Dunn's test
+                                dunn_data = pd.DataFrame({
+                                    'value': data[num_feat],
+                                    'group': data[cat_feat]
+                                })
+                                
+                                # Perform Dunn's test
+                                dunn_results = posthoc_dunn(dunn_data, val_col='value', group_col='group', p_adjust='bonferroni')
+                                
+                                # Store results
+                                results['anova_tests'][f"{num_feat}_by_{cat_feat}"]["dunn_posthoc"] = {
+                                    'p_values': dunn_results.to_dict()
+                                }
+                            except:
+                                # If Dunn's test fails, skip it
+                                pass
+                    except:
+                        # If Kruskal-Wallis test fails, skip it
+                        pass
+                    
+                    # Add to significant differences only if significant with reasonable effect size
+                    if p_value < 0.05 and eta_squared >= 0.01:  # Filter out very small effects
+                        effect_size_category = ""
+                        if eta_squared >= 0.14:
+                            effect_size_category = "strong"
+                        elif eta_squared >= 0.06:
+                            effect_size_category = "moderate"
+                        else:
+                            effect_size_category = "weak"
+                            
+                        # Calculate group means more efficiently
+                        group_means = {group: float(data[data[cat_feat] == group][num_feat].mean()) 
+                                     for group in data[cat_feat].unique()}
+                        
+                        # Get point-biserial correlation if available
+                        point_biserial_info = {}
+                        if f"{num_feat}_by_{cat_feat}" in results['point_biserial_correlations']:
+                            point_biserial_info = results['point_biserial_correlations'][f"{num_feat}_by_{cat_feat}"]
+                            
+                        results['significant_differences'].append({
+                            'numerical_feature': num_feat,
+                            'categorical_feature': cat_feat,
+                            'f_statistic': float(f_stat),
+                            'anova_p_value': float(p_value),
+                            'eta_squared': float(eta_squared),
+                            'omega_squared': float(omega_squared),
+                            'effect_size_category': effect_size_category,
+                            'group_means': group_means,
+                            'point_biserial_correlation': point_biserial_info if point_biserial_info else None,
+                            'equal_variances': results['anova_tests'][f"{num_feat}_by_{cat_feat}"].get("levene_test", {}).get("equal_variances", None)
+                        })
+                        
+                except Exception as e:
+                    # If tests fail, just continue to next pair
+                    continue
+    
+    # Calculate mutual information for only significant pairs found above
+    # This reduces unnecessary calculations
+    if results['significant_differences']:
+        mutual_info = {}
+        sig_pairs = [(d['numerical_feature'], d['categorical_feature']) for d in results['significant_differences']]
+        
+        for num_feat, cat_feat in sig_pairs:
+            # Get clean data
+            data = df_sample[[num_feat, cat_feat]].dropna()
+            
+            if data.empty or data[cat_feat].nunique() <= 1:
+                continue
+            
+            # Discretize numerical feature (needed for mutual information)
+            try:
+                # Use quantiles to discretize into fewer bins for efficiency
+                num_bins = min(5, data[num_feat].nunique())  # Reduced from 10 to 5
+                if num_bins > 1:
+                    data['num_binned'] = pd.qcut(data[num_feat], num_bins, duplicates='drop')
+                    
+                    # Calculate mutual information
+                    mi = mutual_info_score(
+                        data['num_binned'].astype(str).values,
+                        data[cat_feat].astype(str).values
+                    )
+                    mutual_info[f"{num_feat}_vs_{cat_feat}"] = float(mi)
+            except Exception as e:
+                pass
+        
+        # Only add mutual_information if we calculated something
+        if mutual_info:
+            results['mutual_information'] = mutual_info
+    
+    # Sort significant differences by effect size
+    results['significant_differences'].sort(key=lambda x: x['eta_squared'], reverse=True)
+    
+    return results
+
+def generate_summary(results):
+
+    """Generate a summary of key findings from bivariate analysis"""
+    summary = {
+        'strongest_correlations': [],
+        'significant_category_associations': [],
+        'significant_group_differences': [],
+        'key_insights': []
+    }
+    
+    # Extract strongest numerical correlations
+    if 'numerical_vs_numerical' in results and 'significant_correlations' in results['numerical_vs_numerical']:
+        for corr in results['numerical_vs_numerical']['significant_correlations'][:5]:  # Top 5
+            summary['strongest_correlations'].append({
+                'features': f"{corr['feature_1']} and {corr['feature_2']}",
+                'correlation': corr['pearson_correlation'],
+                'type': corr['correlation_type'],
+                'strength': corr['correlation_strength'],
+                'p_value': corr['p_value'],
+                'relationship_type': corr.get('relationship_type', 'linear')
+            })
+    
+    # Extract strongest categorical associations
+    if 'categorical_vs_categorical' in results and 'significant_associations' in results['categorical_vs_categorical']:
+        for assoc in results['categorical_vs_categorical']['significant_associations'][:5]:  # Top 5
+            summary['significant_category_associations'].append({
+                'features': f"{assoc['feature_1']} and {assoc['feature_2']}",
+                'cramer_v': assoc['cramers_v'],
+                'strength': assoc['association_strength'],
+                'p_value': assoc['p_value'],
+                'phi_coefficient': assoc.get('phi_coefficient', None)
+            })
+    
+    # Extract most significant group differences
+    if 'numerical_vs_categorical' in results and 'significant_differences' in results['numerical_vs_categorical']:
+        for diff in results['numerical_vs_categorical']['significant_differences'][:5]:  # Top 5
+            summary['significant_group_differences'].append({
+                'numerical': diff['numerical_feature'],
+                'categorical': diff['categorical_feature'],
+                'eta_squared': diff['eta_squared'],
+                'omega_squared': diff.get('omega_squared', None),
+                'effect_size': diff['effect_size_category'],
+                'anova_p_value': diff['anova_p_value'],
+                'equal_variances': diff.get('equal_variances', None)
+            })
+    
+    # Generate key insights
+    insights = []
+    
+    # Insight from numerical correlations
+    if summary['strongest_correlations']:
+        top_corr = summary['strongest_correlations'][0]
+        insights.append(f"The strongest numerical relationship is between {top_corr['features']} with a {top_corr['strength']} " 
+                        f"{top_corr['type']} correlation of {top_corr['correlation']:.3f} (p={top_corr['p_value']:.4f}).")
+        
+        # Additional insight on relationship type
+        if top_corr.get('relationship_type') == 'potentially non-linear':
+            insights.append(f"The relationship between {top_corr['features']} appears to be non-linear, "
+                           f"as indicated by the difference between Pearson and Spearman correlations.")
+    
+    # Insight from categorical associations
+    if summary['significant_category_associations']:
+        top_assoc = summary['significant_category_associations'][0]
+        insights.append(f"The strongest association between categorical variables is between {top_assoc['features']} " 
+                        f"with a {top_assoc['strength']} relationship (Cramer's V={top_assoc['cramer_v']:.3f}, p={top_assoc['p_value']:.4f}).")
+    
+    # Insight from group differences
+    if summary['significant_group_differences']:
+        top_diff = summary['significant_group_differences'][0]
+        insights.append(f"The categorical variable {top_diff['categorical']} has a {top_diff['effect_size']} effect " 
+                        f"on {top_diff['numerical']} (η²={top_diff['eta_squared']:.3f}, p={top_diff['anova_p_value']:.4f}).")
+        
+        # Add insight about equal variances assumption
+        if top_diff.get('equal_variances') is not None:
+            if top_diff['equal_variances']:
+                insights.append(f"The equal variances assumption is met for {top_diff['numerical']} across {top_diff['categorical']} groups, "
+                               f"supporting the validity of the ANOVA results.")
+            else:
+                insights.append(f"The equal variances assumption is violated for {top_diff['numerical']} across {top_diff['categorical']} groups, "
+                               f"suggesting Welch's ANOVA may be more appropriate.")
+    
+    # Check for data quality issues
+    data_quality_issues = []
+    
+    # Check for potentially misleading relationships
+    if 'numerical_vs_numerical' in results and 'significant_correlations' in results['numerical_vs_numerical']:
+        for corr in results['numerical_vs_numerical']['significant_correlations']:
+            if corr['pearson_correlation'] > 0.9 or corr['pearson_correlation'] < -0.9:
+                data_quality_issues.append(f"The very strong correlation between {corr['feature_1']} and {corr['feature_2']} "
+                                          f"may indicate multicollinearity issues in predictive modeling.")
+    
+    # Check for imbalanced categorical variables
+    if 'numerical_vs_categorical' in results and 'group_statistics' in results['numerical_vs_categorical']:
+        for key, stats in results['numerical_vs_categorical']['group_statistics'].items():
+            if 'counts' in stats:
+                counts = list(stats['counts'].values())
+                if counts and max(counts) / min(counts) > 10:  # Imbalanced if one group is 10x larger than smallest
+                    features = key.split('_by_')
+                    if len(features) == 2:
+                        data_quality_issues.append(f"The categorical variable {features[1]} has highly imbalanced groups "
+                                                  f"which may affect the reliability of {features[0]} analysis.")
+    
+    # Add data quality issues as insights
+    for issue in data_quality_issues[:2]:  # Limit to top 2 issues
+        insights.append(issue)
+    
+    # Add final observations
+    if 'numerical_vs_numerical' in results:
+        # Check for non-linear relationships
+        non_linear_count = 0
+        if 'significant_correlations' in results['numerical_vs_numerical']:
+            for corr in results['numerical_vs_numerical']['significant_correlations']:
+                if corr.get('relationship_type') == 'potentially non-linear':
+                    non_linear_count += 1
+        
+        if non_linear_count > 0:
+            insights.append(f"Found {non_linear_count} potentially non-linear relationships among numerical variables, "
+                          f"suggesting that linear models may not fully capture the complexity of the data.")
+    
+    # Cross-analysis insights
+    has_num_num = 'numerical_vs_numerical' in results and 'significant_correlations' in results['numerical_vs_numerical']
+    has_cat_cat = 'categorical_vs_categorical' in results and 'significant_associations' in results['categorical_vs_categorical']
+    has_num_cat = 'numerical_vs_categorical' in results and 'significant_differences' in results['numerical_vs_categorical']
+    
+    if has_num_num and has_num_cat:
+        insights.append("Both numerical correlations and categorical group differences were detected, "
+                      "suggesting a mix of continuous and segmented relationships in the data.")
+    
+    # Add mention of mutual information if present
+    if 'numerical_vs_categorical' in results and 'mutual_information' in results['numerical_vs_categorical']:
+        mutual_info = results['numerical_vs_categorical']['mutual_information']
+        if mutual_info:
+            # Find highest mutual information score
+            top_mi_pair = max(mutual_info.items(), key=lambda x: x[1])
+            features = top_mi_pair[0].split('_vs_')
+            if len(features) == 2:
+                insights.append(f"The strongest general statistical dependency (mutual information) was found between "
+                              f"{features[0]} and {features[1]} with score {top_mi_pair[1]:.3f}, "
+                              f"capturing both linear and non-linear relationships.")
+    
+    # Overall data structure insight
+    if has_num_num and has_cat_cat and has_num_cat:
+        insights.append("The data shows a complex structure with significant relationships across all variable types, "
+                      "suggesting potential for both feature engineering and dimensionality reduction.")
+    
+    # Add insights to summary
+    summary['key_insights'] = insights
+    
+    return summary
+
+def bivariate_analysis(data_path: str):
+    """
+    Perform a  comprehensive bivariate analysis on a dataset.
+    
+    Args:
+        data_path: Path to a data file or a pandas DataFrame
+    Returns:
+        A dictionary containing all analysis results
+    """
+    try:
+        if isinstance(data_path, str):
+            data = pd.read_csv(data_path)
+        else:
+            logger.error(f"Unsupported file format: {data_path}")
+            return {"error": f"Unsupported file format: {data_path}"}
+
+        if not isinstance(data, pd.DataFrame):
+            logger.error(f"Input is not a valid pandas DataFrame")
+            return {"error": "Input is not a valid pandas DataFrame"}
+
+        logger.warn("Staring Analysis.....")
+
+        results = comprehensive_bivariate_analysis(df = data)
+        summary = generate_summary(results=results)
+
+        serialized_summary = json.dumps(summary, default=lambda o: o.item() if isinstance(o, np.generic) else str(o))
+
+        return serialized_summary
+        
+    except Exception as e:
+        logger.error(f"Error in analyze_dataset: {str(e)}")
+        return {"error": str(e)} 

src/app/pipelines/eda/tools/analysis_tools/multivariate_analysis.py

@@ -0,0 +1,1039 @@
+import pandas as pd
+import numpy as np
+from scipy import stats
+import statsmodels.api as sm
+from statsmodels.stats.outliers_influence import variance_inflation_factor
+from sklearn.preprocessing import StandardScaler
+from sklearn.decomposition import PCA, FactorAnalysis
+from sklearn.manifold import TSNE, MDS
+from sklearn.cluster import KMeans, AgglomerativeClustering, DBSCAN
+from sklearn.mixture import GaussianMixture
+from sklearn.metrics import silhouette_score, calinski_harabasz_score, davies_bouldin_score
+from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
+from sklearn.feature_selection import mutual_info_classif, mutual_info_regression, f_classif, f_regression, chi2
+from sklearn.feature_selection import SelectKBest, RFE
+from sklearn.neighbors import LocalOutlierFactor
+from sklearn.ensemble import IsolationForest
+from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
+from sklearn.model_selection import cross_val_score
+from sklearn.feature_selection import VarianceThreshold
+from agno.utils.log import logger
+import warnings
+import json
+
+warnings.filterwarnings("ignore")
+
+def preprocess_dataframe(df, target_column=None, categorical_columns=None, verbose=True):
+    """
+    Preprocess a dataframe by identifying categorical columns, handling missing values,
+    and separating the target variable if specified.
+    
+    Parameters:
+    -----------
+    df : pandas.DataFrame
+        The dataframe to preprocess
+    target_column : str, optional
+        The name of the target variable column (if any)
+    categorical_columns : list, optional
+        List of categorical column names (will be auto-detected if None)
+    verbose : bool, default=True
+        Whether to print detailed messages during preprocessing
+    
+    Returns:
+    --------
+    dict
+        A dictionary containing the preprocessed data and metadata
+    """
+    # Create a deep copy to avoid modifying the original dataframe
+    data = df.copy()
+    
+    # Initialize results dictionary to store preprocessing info
+    results = {}
+    
+    # Separate target if specified
+    target = None
+    
+    if target_column is not None and target_column in data.columns:
+        target = data[target_column].copy()
+        data = data.drop(columns=[target_column])
+        target_is_numeric = pd.api.types.is_numeric_dtype(target)
+        results['target_column'] = target_column
+        results['target_is_numeric'] = target_is_numeric
+        if verbose:
+            logger.warning(f"Target variable '{target_column}' detected as {'numeric' if target_is_numeric else 'categorical'}")
+    
+    # Identify categorical and numerical columns
+    if categorical_columns is None:
+        categorical_columns = []
+        for col in data.columns:
+            if pd.api.types.is_object_dtype(data[col]) or pd.api.types.is_categorical_dtype(data[col]) or len(data[col].unique()) < 10:
+                categorical_columns.append(col)
+    
+    numerical_columns = [col for col in data.columns if col not in categorical_columns]
+    
+    results['categorical_columns'] = categorical_columns
+    results['numerical_columns'] = numerical_columns
+    
+    if verbose:
+        logger.warning(f"Detected {len(numerical_columns)} numerical columns and {len(categorical_columns)} categorical columns")
+    
+    # Handle missing values
+    missing_data = data.isnull().sum()
+    results['missing_values'] = {col: count for col, count in missing_data.items() if count > 0}
+    results['missing_values_percentage'] = {col: count/len(data)*100 for col, count in missing_data.items() if count > 0}
+    
+    # For analysis, we'll do simple imputation to handle missing values
+    for col in numerical_columns:
+        if data[col].isnull().sum() > 0:
+            data[col].fillna(data[col].median(), inplace=True)
+    
+    for col in categorical_columns:
+        if data[col].isnull().sum() > 0:
+            data[col].fillna(data[col].mode()[0], inplace=True)
+    
+    # Create a dataframe of numerical data only for correlation analysis
+    numerical_data = data[numerical_columns].copy()
+    
+    # Standardize numerical data
+    scaler = StandardScaler()
+    if len(numerical_columns) > 0:
+        scaled_data = scaler.fit_transform(numerical_data)
+        results['preprocessing'] = {
+            'scaler_mean': scaler.mean_.tolist(),
+            'scaler_scale': scaler.scale_.tolist()
+        }
+    else:
+        scaled_data = np.array([])
+    
+    results['data'] = data
+    results['numerical_data'] = numerical_data
+    results['scaled_data'] = scaled_data
+    results['target'] = target
+    
+    return results
+
+def analyze_correlations(numerical_data, verbose=True):
+    """
+    Perform correlation analysis on numerical data.
+    
+    Parameters:
+    -----------
+    numerical_data : pandas.DataFrame
+        DataFrame containing only numerical columns
+    verbose : bool, default=True
+        Whether to print detailed messages during analysis
+    
+    Returns:
+    --------
+    dict
+        A dictionary containing correlation analysis results
+    """
+    if len(numerical_data.columns) <= 1:
+        if verbose:
+            logger.warning("Correlation analysis requires at least 2 numerical columns")
+        return {}
+    
+    correlation_results = {}
+    
+    # Generate correlation matrix
+    correlation_matrix = numerical_data.corr()
+    correlation_results['matrix'] = correlation_matrix
+    
+    # Identify highly correlated features
+    high_corr_pairs = []
+    for i in range(len(correlation_matrix.columns)):
+        for j in range(i+1, len(correlation_matrix.columns)):
+            col1 = correlation_matrix.columns[i]
+            col2 = correlation_matrix.columns[j]
+            corr_value = correlation_matrix.iloc[i, j]
+            if abs(corr_value) > 0.7:
+                high_corr_pairs.append((col1, col2, corr_value))
+    
+    correlation_results['high_correlation_pairs'] = high_corr_pairs
+    
+    # Multicollinearity using VIF (Only for numeric predictors)
+    if len(numerical_data.columns) > 1:
+        try:
+            X = sm.add_constant(numerical_data)
+            vif_data = pd.DataFrame()
+            vif_data["Variable"] = X.columns
+            vif_data["VIF"] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]
+            correlation_results['vif'] = vif_data
+            
+            # Identify multicollinearity issues
+            multicollinearity_issues = vif_data[vif_data["VIF"] > 10].to_dict('records')
+            correlation_results['multicollinearity_issues'] = multicollinearity_issues
+        except Exception as e:
+            if verbose:
+                logger.error(f"VIF calculation failed: {str(e)}")
+    
+    return correlation_results
+
+def perform_dimensionality_reduction(scaled_data, numerical_columns, data_size, verbose=True):
+    """
+    Perform various dimensionality reduction techniques on scaled numerical data.
+    
+    Parameters:
+    -----------
+    scaled_data : numpy.ndarray
+        Standardized numerical data
+    numerical_columns : list
+        List of numerical column names
+    data_size : int
+        Number of rows in the dataset
+    verbose : bool, default=True
+        Whether to print detailed messages during analysis
+    
+    Returns:
+    --------
+    dict
+        A dictionary containing dimensionality reduction results
+    """
+    if len(numerical_columns) <= 1:
+        if verbose:
+            logger.warning("Dimensionality reduction requires at least 2 numerical columns")
+        return {}
+    
+    dr_results = {}
+    
+    # PCA
+    try:
+        pca = PCA(random_state=42)
+        pca_results = pca.fit_transform(scaled_data)
+        
+        # Explained variance
+        explained_variance = pca.explained_variance_ratio_
+        cumulative_variance = np.cumsum(explained_variance)
+        
+        # Find components that explain at least 80% of variance
+        components_80pct = np.argmax(cumulative_variance >= 0.8) + 1
+        
+        # Component loadings
+        loadings = pd.DataFrame(
+            pca.components_.T,
+            columns=[f'PC{i+1}' for i in range(pca.n_components_)],
+            index=numerical_columns
+        )
+        
+        dr_results['pca'] = {
+            'explained_variance_ratio': explained_variance.tolist(),
+            'cumulative_variance': cumulative_variance.tolist(),
+            'components_for_80_percent_variance': components_80pct,
+            'component_loadings': loadings,
+            'pca_object': pca
+        }
+        
+        # Factor Analysis
+        if len(numerical_columns) >= 3:
+            try:
+                factor = FactorAnalysis(n_components=min(5, len(numerical_columns)), random_state=42)
+                factor.fit(scaled_data)
+                
+                loadings = pd.DataFrame(
+                    factor.components_.T,
+                    columns=[f'Factor{i+1}' for i in range(factor.n_components)],
+                    index=numerical_columns
+                )
+                
+                dr_results['factor_analysis'] = {
+                    'loadings': loadings,
+                    'factor_object': factor
+                }
+            except Exception as e:
+                if verbose:
+                    logger.error(f"Factor Analysis failed: {str(e)}")
+        
+        # t-SNE (for datasets that aren't too large)
+        if data_size <= 5000:
+            try:
+                tsne = TSNE(n_components=2, random_state=42)
+                tsne_results = tsne.fit_transform(scaled_data)
+                
+                dr_results['tsne'] = {
+                    'coordinates': tsne_results,
+                    'tsne_object': tsne
+                }
+            except Exception as e:
+                if verbose:
+                    logger.error(f"t-SNE failed: {str(e)}")
+        
+        # MDS (for smaller datasets)
+        if data_size <= 2000:
+            try:
+                mds = MDS(n_components=2, random_state=42)
+                mds_results = mds.fit_transform(scaled_data)
+                
+                dr_results['mds'] = {
+                    'coordinates': mds_results,
+                    'mds_object': mds
+                }
+            except Exception as e:
+                if verbose:
+                    logger.error(f"MDS failed: {str(e)}")
+    
+    except Exception as e:
+        if verbose:
+            logger.error(f"PCA failed: {str(e)}")
+    
+    return dr_results
+
+def perform_cluster_analysis(scaled_data, numerical_data, data_size, verbose=True):
+    """
+    Perform various clustering techniques on scaled numerical data.
+    
+    Parameters:
+    -----------
+    scaled_data : numpy.ndarray
+        Standardized numerical data
+    numerical_data : pandas.DataFrame
+        DataFrame containing numerical data (for cluster analysis)
+    data_size : int
+        Number of rows in the dataset
+    verbose : bool, default=True
+        Whether to print detailed messages during analysis
+    
+    Returns:
+    --------
+    dict
+        A dictionary containing clustering results
+    """
+    if len(numerical_data.columns) <= 1 or data_size <= 10:
+        if verbose:
+            logger.warning("Clustering requires at least 2 numerical columns and more than 10 data points")
+        return {}
+    
+    clustering_results = {}
+    
+    # K-means clustering
+    try:
+        # Determine optimal number of clusters using silhouette score
+        sil_scores = []
+        max_clusters = min(10, data_size // 10)  # reasonable max number of clusters
+        
+        for k in range(2, max_clusters + 1):
+            kmeans = KMeans(n_clusters=k, random_state=42)
+            labels = kmeans.fit_predict(scaled_data)
+            sil_score = silhouette_score(scaled_data, labels)
+            sil_scores.append((k, sil_score))
+        
+        # Find the best k
+        best_k = max(sil_scores, key=lambda x: x[1])[0]
+        
+        # Run K-means with optimal k
+        kmeans = KMeans(n_clusters=best_k, random_state=42)
+        labels = kmeans.fit_predict(scaled_data)
+        
+        # Calculate cluster evaluation metrics
+        sil_avg = silhouette_score(scaled_data, labels)
+        ch_score = calinski_harabasz_score(scaled_data, labels)
+        db_score = davies_bouldin_score(scaled_data, labels)
+        
+        # Analyze cluster characteristics
+        cluster_analysis = {}
+        for cluster in range(best_k):
+            cluster_data = numerical_data.iloc[labels == cluster]
+            cluster_analysis[f'Cluster_{cluster}'] = {
+                'size': len(cluster_data),
+                'percentage': len(cluster_data) / data_size * 100,
+                'mean': cluster_data.mean().to_dict(),
+                'std': cluster_data.std().to_dict()
+            }
+        
+        clustering_results['kmeans'] = {
+            'best_k': best_k,
+            'silhouette_scores': dict(sil_scores),
+            'silhouette_avg': sil_avg,
+            'calinski_harabasz_score': ch_score,
+            'davies_bouldin_score': db_score,
+            'cluster_analysis': cluster_analysis,
+            'kmeans_object': kmeans
+        }
+        
+        # DBSCAN for density-based clustering
+        if data_size <= 5000:  # DBSCAN can be slow on large datasets
+            try:
+                dbscan = DBSCAN(eps=0.5, min_samples=min(5, data_size // 100))
+                dbscan_labels = dbscan.fit_predict(scaled_data)
+                
+                # Count number of clusters and noise points
+                n_clusters = len(set(dbscan_labels)) - (1 if -1 in dbscan_labels else 0)
+                n_noise = list(dbscan_labels).count(-1)
+                
+                # Only calculate silhouette if we have clusters and not all points are noise
+                if n_clusters > 1 and n_noise < data_size:
+                    valid_data = scaled_data[dbscan_labels != -1]
+                    valid_labels = dbscan_labels[dbscan_labels != -1]
+                    if len(set(valid_labels)) > 1:
+                        db_sil_score = silhouette_score(valid_data, valid_labels)
+                    else:
+                        db_sil_score = None
+                else:
+                    db_sil_score = None
+                
+                clustering_results['dbscan'] = {
+                    'n_clusters': n_clusters,
+                    'n_noise': n_noise,
+                    'silhouette_score': db_sil_score,
+                    'dbscan_object': dbscan
+                }
+            except Exception as e:
+                if verbose:
+                    logger.error(f"DBSCAN failed: {str(e)}")
+        
+        # Gaussian Mixture Models
+        try:
+            gmm = GaussianMixture(n_components=best_k, random_state=42)
+            gmm_labels = gmm.fit_predict(scaled_data)
+            
+            gmm_sil_score = silhouette_score(scaled_data, gmm_labels)
+            gmm_ch_score = calinski_harabasz_score(scaled_data, gmm_labels)
+            gmm_db_score = davies_bouldin_score(scaled_data, gmm_labels)
+            
+            clustering_results['gmm'] = {
+                'n_components': best_k,
+                'silhouette_score': gmm_sil_score,
+                'calinski_harabasz_score': gmm_ch_score,
+                'davies_bouldin_score': gmm_db_score,
+                'gmm_object': gmm
+            }
+        except Exception as e:
+            if verbose:
+                logger.error(f"GMM failed: {str(e)}")
+    
+    except Exception as e:
+        if verbose:
+            logger.error(f"Clustering analysis failed: {str(e)}")
+    
+    return clustering_results
+
+def analyze_feature_importance(data, target, numerical_columns, categorical_columns, target_is_numeric, verbose=True):
+    """
+    Analyze feature importance using various techniques.
+    
+    Parameters:
+    -----------
+    data : pandas.DataFrame
+        The preprocessed dataframe
+    target : pandas.Series
+        The target variable
+    numerical_columns : list
+        List of numerical column names
+    categorical_columns : list
+        List of categorical column names
+    target_is_numeric : bool
+        Whether the target variable is numeric
+    verbose : bool, default=True
+        Whether to print detailed messages during analysis
+    
+    Returns:
+    --------
+    dict
+        A dictionary containing feature importance analysis results
+    """
+    if target is None:
+        if verbose:
+            logger.warning("Feature importance analysis requires a target variable")
+        return {}
+    
+    feature_importance_results = {}
+    
+    # Prepare data for feature importance
+    X = data.copy()
+    y = target.copy()
+    
+    # Encode categorical features for feature importance
+    X_encoded = pd.get_dummies(X, columns=categorical_columns, drop_first=True)
+    
+    # Univariate feature importance
+    try:
+        if target_is_numeric:
+            # Numeric target: use F-test for numeric features
+            selector = SelectKBest(score_func=f_regression, k='all')
+            selector.fit(X_encoded, y)
+            f_scores = pd.Series(selector.scores_, index=X_encoded.columns)
+            p_values = pd.Series(selector.pvalues_, index=X_encoded.columns)
+            
+            # Mutual information
+            try:
+                var_thresh = VarianceThreshold(threshold=0.01) 
+                X_encoded = pd.DataFrame(var_thresh.fit_transform(X_encoded), columns=X_encoded.columns[var_thresh.get_support()])
+                mi_scores = mutual_info_regression(X_encoded, y)
+                mi_series = pd.Series(mi_scores, index=X_encoded.columns)
+                feature_importance_results['mutual_info'] = mi_series.sort_values(ascending=False).to_dict()
+            except Exception as e:
+                if verbose:
+                    logger.error(f"Mutual information calculation failed: {str(e)}")
+            
+            feature_importance_results['f_regression'] = {
+                'scores': f_scores.sort_values(ascending=False).to_dict(),
+                'p_values': p_values.sort_values().to_dict()
+            }
+        else:
+            # Categorical target: use chi2 for numeric features
+            # Need non-negative features for chi2
+            X_chi = X_encoded.copy()
+            for col in X_chi.columns:
+                if X_chi[col].min() < 0:
+                    X_chi[col] = X_chi[col] - X_chi[col].min()
+            
+            selector = SelectKBest(score_func=chi2, k='all')
+            selector.fit(X_chi, y)
+            chi2_scores = pd.Series(selector.scores_, index=X_encoded.columns)
+            p_values = pd.Series(selector.pvalues_, index=X_encoded.columns)
+            
+            # F-test for classification
+            f_selector = SelectKBest(score_func=f_classif, k='all')
+            f_selector.fit(X_encoded, y)
+            f_scores = pd.Series(f_selector.scores_, index=X_encoded.columns)
+            f_p_values = pd.Series(f_selector.pvalues_, index=X_encoded.columns)
+            
+            # Mutual information
+            try:
+                mi_scores = mutual_info_classif(X_encoded, y)
+                mi_series = pd.Series(mi_scores, index=X_encoded.columns)
+                feature_importance_results['mutual_info'] = mi_series.sort_values(ascending=False).to_dict()
+            except Exception as e:
+                if verbose:
+                    logger.error(f"Mutual information calculation failed: {str(e)}")
+            
+            feature_importance_results['chi2'] = {
+                'scores': chi2_scores.sort_values(ascending=False).to_dict(),
+                'p_values': p_values.sort_values().to_dict()
+            }
+            
+            feature_importance_results['f_classif'] = {
+                'scores': f_scores.sort_values(ascending=False).to_dict(),
+                'p_values': f_p_values.sort_values().to_dict()
+            }
+        
+        # Tree-based feature importance
+        try:
+            if target_is_numeric:
+                model = RandomForestRegressor(n_estimators=100, random_state=42)
+            else:
+                model = RandomForestClassifier(n_estimators=100, random_state=42)
+            
+            model.fit(X_encoded, y)
+            importances = pd.Series(model.feature_importances_, index=X_encoded.columns)
+            
+            feature_importance_results['random_forest'] = importances.sort_values(ascending=False).to_dict()
+            
+            # Recursive Feature Elimination
+            if len(X_encoded.columns) > 5:
+                rfe = RFE(estimator=model, n_features_to_select=min(10, len(X_encoded.columns)), step=1)
+                rfe.fit(X_encoded, y)
+                rfe_ranking = pd.Series(rfe.ranking_, index=X_encoded.columns)
+                
+                feature_importance_results['rfe'] = {
+                    'selected_features': X_encoded.columns[rfe.support_].tolist(),
+                    'feature_ranking': rfe_ranking.sort_values().to_dict()
+                }
+        except Exception as e:
+            if verbose:
+                logger.error(f"Tree-based feature importance calculation failed: {str(e)}")
+    
+    except Exception as e:
+        if verbose:
+            logger.error(f"Feature importance analysis failed: {str(e)}")
+    
+    return feature_importance_results
+
+def detect_outliers(scaled_data, data_size, verbose=True):
+    """
+    Detect multivariate outliers using different techniques.
+    
+    Parameters:
+    -----------
+    scaled_data : numpy.ndarray
+        Standardized numerical data
+    data_size : int
+        Number of rows in the dataset
+    verbose : bool, default=True
+        Whether to print detailed messages during analysis
+    
+    Returns:
+    --------
+    dict
+        A dictionary containing outlier detection results
+    """
+    if scaled_data.size == 0 or scaled_data.shape[1] <= 1:
+        if verbose:
+            logger.warning("Outlier detection requires at least 2 numerical columns")
+        return {}
+    
+    outlier_results = {}
+    
+    # Isolation Forest
+    try:
+        iso = IsolationForest(contamination=0.05, random_state=42)
+        outliers_isof = iso.fit_predict(scaled_data)
+        outlier_indices_isof = np.where(outliers_isof == -1)[0]
+        
+        outlier_results['isolation_forest'] = {
+            'num_outliers': len(outlier_indices_isof),
+            'outlier_percentage': len(outlier_indices_isof) / data_size * 100
+        }
+    except Exception as e:
+        if verbose:
+            logger.error(f"Isolation Forest failed: {str(e)}")
+    
+    # Local Outlier Factor (for smaller datasets)
+    if data_size <= 5000:
+        try:
+            lof = LocalOutlierFactor(n_neighbors=20, contamination=0.05)
+            outliers_lof = lof.fit_predict(scaled_data)
+            outlier_indices_lof = np.where(outliers_lof == -1)[0]
+            
+            outlier_results['local_outlier_factor'] = {
+                'num_outliers': len(outlier_indices_lof),
+                'outlier_indices': outlier_indices_lof.tolist(),
+                'outlier_percentage': len(outlier_indices_lof) / data_size * 100
+            }
+        except Exception as e:
+            if verbose:
+                logger.error(f"Local Outlier Factor failed: {str(e)}")
+    
+    return outlier_results
+
+def perform_statistical_testing(data, target, scaled_data, numerical_columns, categorical_columns, target_is_numeric, verbose=True):
+    """
+    Perform various statistical tests on the data.
+    
+    Parameters:
+    -----------
+    data : pandas.DataFrame
+        The preprocessed dataframe
+    target : pandas.Series
+        The target variable
+    scaled_data : numpy.ndarray
+        Standardized numerical data
+    numerical_columns : list
+        List of numerical column names
+    categorical_columns : list
+        List of categorical column names
+    target_is_numeric : bool
+        Whether the target variable is numeric
+    verbose : bool, default=True
+        Whether to print detailed messages during analysis
+    
+    Returns:
+    --------
+    dict
+        A dictionary containing statistical testing results
+    """
+    if target is None:
+        if verbose:
+            print("Statistical testing requires a target variable")
+        return {}
+    
+    statistical_testing_results = {}
+    
+    # MANOVA (only for multivariate numeric data with categorical target)
+    if not target_is_numeric and len(numerical_columns) > 1:
+        try:
+            unique_classes = target.unique()
+            if len(unique_classes) > 1 and len(unique_classes) <= 10:
+                # Prepare groups for MANOVA
+                numerical_data = data[numerical_columns]
+                groups = [numerical_data.iloc[target == val].values for val in unique_classes]
+                
+                # Run MANOVA
+                manova_result = stats.manova(groups)
+                
+                statistical_testing_results['manova'] = {
+                    'test_statistic': float(manova_result.statistic),
+                    'p_value': float(manova_result.pvalue),
+                    'significant': float(manova_result.pvalue) < 0.05
+                }
+        except Exception as e:
+            if verbose:
+                print(f"MANOVA failed: {str(e)}")
+    
+    # Linear Discriminant Analysis (for classification problems)
+    if not target_is_numeric:
+        try:
+            lda = LinearDiscriminantAnalysis()
+            X_lda = lda.fit_transform(scaled_data, target)
+            
+            statistical_testing_results['lda'] = {
+                'explained_variance_ratio': lda.explained_variance_ratio_.tolist(),
+                'coordinates': X_lda.tolist(),
+                'lda_object': lda
+            }
+        except Exception as e:
+            if verbose:
+                print(f"LDA failed: {str(e)}")
+    
+    # # Multivariate regression for numeric target
+    # if target_is_numeric:
+    #     # try:
+    #         for col in numerical_columns:
+    #             data[col] = pd.to_numeric(data[col], errors='coerce')
+
+    #         data[numerical_columns] = data[numerical_columns].fillna(data[numerical_columns].mean())
+
+    #         X_encoded = pd.get_dummies(data[categorical_columns], drop_first=True)
+
+    #         X_final = pd.concat([data[numerical_columns], X_encoded], axis=1)
+    #         target = pd.to_numeric(target, errors='coerce').fillna(target.mean())
+
+    #         X_final = X_final.apply(pd.to_numeric, errors='coerce')
+    #         X_final = X_final.fillna(0)  # optional: or use mean imputation
+
+    #         X_final = X_final.astype('float64')
+    #         target = target.astype('float64')
+
+    #         X_sm = sm.add_constant(X_final)
+
+    #         # Fit model
+    #         model = sm.OLS(target, X_sm).fit()
+
+    #         statistical_testing_results['multivariate_regression'] = {
+    #             'r_squared': model.rsquared,
+    #             'adj_r_squared': model.rsquared_adj,
+    #             'f_statistic': float(model.fvalue),
+    #             'f_pvalue': float(model.f_pvalue),
+    #             'coefficients': model.params.to_dict(),
+    #             'p_values': model.pvalues.to_dict(),
+    #             'significant_features': [feature for feature, p_val in model.pvalues.items() if p_val < 0.05]
+    #         }
+    #     # except Exception as e:
+    #         if verbose:
+    #             print(f"Multivariate regression failed: {str(e)}")
+
+    return statistical_testing_results
+
+def generate_summary(results, df, verbose=True):
+    """
+    Generate a summary of analysis results.
+    
+    Parameters:
+    -----------
+    results : dict
+        Dictionary containing all analysis results
+    df : pandas.DataFrame
+        The original dataframe
+    verbose : bool, default=True
+        Whether to print the summary
+    
+    Returns:
+    --------
+    dict
+        A dictionary containing the summary
+    """
+    summary = {}
+    
+    summary['dataset'] = {
+        'rows': len(df),
+        'columns': len(df.columns),
+        'numerical_features': len(results.get('numerical_columns', [])),
+        'categorical_features': len(results.get('categorical_columns', [])),
+        'missing_values': sum(df.isnull().sum() > 0)
+    }
+    
+    if 'correlation' in results:
+        high_corr_pairs = results['correlation'].get('high_correlation_pairs', [])
+        high_corr_count = len(high_corr_pairs)
+        multicollinearity_issues = results['correlation'].get('multicollinearity_issues', [])
+
+        summary['correlation'] = {
+            'highly_correlated_pairs': high_corr_count,
+            'correlation_threshold': 0.7,
+            'top_correlations': high_corr_pairs[:5] if high_corr_pairs else [],
+            'multicollinearity_issues': len(multicollinearity_issues),
+            'vif_threshold': 10
+        }
+    
+    # Dimensionality reduction summary
+    if 'dimensionality_reduction' in results and 'pca' in results['dimensionality_reduction']:
+        pca_results = results['dimensionality_reduction']['pca']
+        
+        summary['dimensionality_reduction'] = {
+            'components_for_80_percent_variance': pca_results['components_for_80_percent_variance'],
+            'total_components': len(pca_results['explained_variance_ratio']),
+            'first_component_variance': pca_results['explained_variance_ratio'][0] * 100
+        }
+    
+    # Clustering summary
+    if 'clustering' in results and 'kmeans' in results['clustering']:
+        kmeans_results = results['clustering']['kmeans']
+        
+        summary['clustering'] = {
+            'optimal_clusters': kmeans_results['best_k'],
+            'silhouette_score': kmeans_results['silhouette_avg'],
+            'clearly_separable': kmeans_results['silhouette_avg'] > 0.5
+        }
+    
+    # Feature importance summary
+    if 'feature_importance' in results:
+        top_features_with_explanations = []
+        
+        # Get top features from RF if available
+        if 'random_forest' in results['feature_importance']:
+            rf_importances = results['feature_importance']['random_forest']
+            top_rf = sorted(rf_importances.items(), key=lambda x: x[1], reverse=True)[:5]
+            for feature, importance in top_rf:
+                explanation = f"{feature}: Importance {importance:.3f} (Random Forest: relative feature contribution to prediction accuracy)."
+                top_features_with_explanations.append(explanation)
+        
+        # Get top features from mutual info if available
+        elif 'mutual_info' in results['feature_importance']:
+            mi_importances = results['feature_importance']['mutual_info']
+            top_mi = sorted(mi_importances.items(), key=lambda x: x[1], reverse=True)[:5]
+            for feature, importance in top_mi:
+                explanation = f"{feature}: Importance {importance:.3f} (Mutual Information: degree of dependency with target)."
+                top_features_with_explanations.append(explanation)
+        
+        summary['feature_importance'] = {
+            'top_features': top_features_with_explanations
+        }
+    
+    # Outlier summary
+    if 'outlier_detection' in results and 'isolation_forest' in results['outlier_detection']:
+        iso_results = results['outlier_detection']['isolation_forest']
+        
+        summary['outliers'] = {
+            'percentage': iso_results['outlier_percentage'],
+            'count': iso_results['num_outliers']
+        }
+    
+    # Statistical testing summary
+    if 'statistical_testing' in results:
+        stat_tests = []
+        
+        if 'manova' in results['statistical_testing']:
+            significant = results['statistical_testing']['manova']['significant']
+            stat_tests.append(f"MANOVA: {'Significant' if significant else 'Not significant'}")
+        
+        if 'lda' in results['statistical_testing']:
+            stat_tests.append("LDA performed")
+        
+        if 'multivariate_regression' in results['statistical_testing']:
+            reg_results = results['statistical_testing']['multivariate_regression']
+            r_squared = reg_results['r_squared']
+            significant_features = len(reg_results['significant_features'])
+            stat_tests.append(f"Regression: R² = {r_squared:.3f}, Significant features: {significant_features}")
+        
+        summary['statistical_tests'] = stat_tests
+    
+        output_string = ""
+    
+    output_string = ""
+    
+    if verbose:
+        output_string += "\n=== MULTIVARIATE ANALYSIS SUMMARY ===\n"
+        output_string += f"Dataset: {summary['dataset']['rows']} rows, {summary['dataset']['columns']} columns\n"
+        output_string += f"Features: {summary['dataset']['numerical_features']} numerical, {summary['dataset']['categorical_features']} categorical\n"
+        
+        if 'correlation' in summary:
+            output_string += f"Correlations (Threshold: {summary['correlation']['correlation_threshold']}):\n"
+            if summary['correlation']['highly_correlated_pairs'] > 0:
+                for col1, col2, corr_value in summary['correlation']['top_correlations']:
+                    output_string += f"  - {col1} & {col2}: {corr_value:.3f}\n"
+            else:
+                output_string += "  - No highly correlated feature pairs found.\n"
+            
+            output_string += f"Multicollinearity (VIF Threshold: {summary['correlation']['vif_threshold']}): {summary['correlation']['multicollinearity_issues']} issues\n"
+        
+        if 'dimensionality_reduction' in summary and pca_results:
+            output_string += f"\nPCA: {pca_results['components_for_80_percent_variance']} components explain 80% of variance\n"
+            output_string += f"First component explains {pca_results['explained_variance_ratio'][0] * 100:.2f}% of variance\n"
+            
+            output_string += "\nComponent Loadings:\n"
+            output_string += str(pca_results['component_loadings']) + "\n"
+            
+            output_string += "\nComponent Interpretation:\n"
+            for pc in pca_results['component_loadings'].columns:
+                output_string += f"  - {pc}:\n"
+                top_features = pca_results['component_loadings'][pc].abs().sort_values(ascending=False).head(3)
+                for feature, loading in top_features.items():
+                    output_string += f"    {feature}: {loading:.3f}\n"
+                output_string += "-" * 10 + "\n"
+            
+            output_string += "\nPCA Meaning:\n"
+            output_string += "  - PCA has reduced the dimensionality of the data while retaining 80% of the variance.\n"
+            output_string += "  - The components represent combinations of the original variables, with loadings indicating the strength and direction of each variable's influence on the component.\n"
+            output_string += "  - The top loading features for each component can help in understanding what the components represent.\n"
+        
+        if 'clustering' in summary and kmeans_results:
+            output_string += f"\nOptimal clusters: {kmeans_results['best_k']}\n"
+            output_string += f"Cluster separation: {kmeans_results['silhouette_avg']:.3f} (silhouette score)\n"
+            
+            output_string += "\nCluster Descriptions:\n"
+            for cluster, analysis in kmeans_results['cluster_analysis'].items():
+                output_string += f"  - {cluster}: Size={analysis['size']} ({analysis['percentage']:.2f}%)\n"
+                output_string += "    Mean values:\n"
+                for feature, mean_val in analysis['mean'].items():
+                    output_string += f"      {feature}: {mean_val:.3f}\n"
+                output_string += "    Standard deviations:\n"
+                for feature, std_val in analysis['std'].items():
+                    output_string += f"      {feature}: {std_val:.3f}\n"
+                output_string += "-" * 20 + "\n"
+            
+            output_string += "\nCluster Differences:\n"
+            if kmeans_results['best_k'] > 1:
+                feature_means = {}
+                for cluster, analysis in kmeans_results['cluster_analysis'].items():
+                    for feature, mean_val in analysis['mean'].items():
+                        if feature not in feature_means:
+                            feature_means[feature] = {}
+                        feature_means[feature][cluster] = mean_val
+                
+                for feature, cluster_means in feature_means.items():
+                    output_string += f"  - {feature}:\n"
+                    for cluster, mean_val in cluster_means.items():
+                        output_string += f"    {cluster}: {mean_val:.3f}\n"
+                    
+                    mean_values = list(cluster_means.values())
+                    if len(set(mean_values)) > 1:
+                        max_diff = max(mean_values) - min(mean_values)
+                        output_string += f"    Max difference: {max_diff:.3f}\n"
+                    output_string += "-" * 10 + "\n"
+            
+            output_string += "\nClustering Meaning:\n"
+            output_string += "  - The clusters represent distinct groups within the data, characterized by differences in the mean values of the numerical features.\n"
+            if kmeans_results['silhouette_avg'] > 0.5:
+                output_string += "  - The high silhouette score indicates that the clusters are well-separated and meaningful.\n"
+            else:
+                output_string += "  - The silhouette score suggests the clusters are reasonably separated.\n"
+        
+        if 'feature_importance' in summary:
+            output_string += "Top Features:\n"
+            for feature_explanation in summary['feature_importance']['top_features']:
+                output_string += f"  - {feature_explanation}\n"
+        
+        if 'outliers' in summary:
+            output_string += f"Outliers: {summary['outliers']['count']} ({summary['outliers']['percentage']:.2f}%)\n"
+        
+        if 'statistical_tests' in summary:
+            output_string += "Statistical tests:\n"
+            for test in summary['statistical_tests']:
+                output_string += f"  - {test}\n"
+                
+    return output_string
+
+def safe_prepare(results):
+    cleaned_results = []
+    for item in results:
+        if item is None:
+            continue  
+        elif isinstance(item, dict):
+            cleaned_results.append(json.dumps(item))  
+        elif isinstance(item, (list, str)):
+            cleaned_results.append(item)
+        else:
+            cleaned_results.append(str(item))
+    return json.dumps(cleaned_results)
+
+def multivariate_analysis(data_path: str, target_column_name: str):
+    """
+    Perform comprehensive multivariate analysis on a dataset from the specified file path.
+    
+    Parameters:
+    -----------
+    data_path : str
+        Path to the data file (supports CSV, Excel, etc. based on pandas read capabilities)
+    target_column_name : str
+        Name of the target column / feature
+    
+    Returns:
+    --------
+    dict
+        A dictionary containing all analysis results
+    """
+    target_column = target_column_name
+    # target_column='Transported' 
+    categorical_columns = None
+    verbose=True
+
+    try:
+        # Determine file type and read data
+        if verbose:
+            logger.warning(f"Reading data from {data_path}...")
+        
+        if data_path.endswith('.csv'):
+            df = pd.read_csv(data_path)
+        elif data_path.endswith(('.xls', '.xlsx')):
+            df = pd.read_excel(data_path)
+        elif data_path.endswith('.json'):
+            df = pd.read_json(data_path)
+        elif data_path.endswith('.parquet'):
+            df = pd.read_parquet(data_path)
+        else:
+            raise ValueError("Unsupported file format. Please provide a CSV, Excel, JSON, or Parquet file.")
+        
+        if verbose:
+            logger.warning(f"Data loaded successfully. Shape: {df.shape}")
+        
+        results = {}
+        
+        if verbose:
+            logger.warning("Preprocessing data...")
+        
+        preprocess_results = preprocess_dataframe(df, target_column, categorical_columns, verbose)
+        results.update(preprocess_results)
+        
+        data = results['data']
+        numerical_data = results['numerical_data']
+        scaled_data = results['scaled_data']
+        target = results['target']
+        numerical_columns = results['numerical_columns']
+        categorical_columns = results['categorical_columns']
+        target_is_numeric = results.get('target_is_numeric', None)
+        
+        if verbose:
+            logger.warning("Analyzing correlations...")
+        
+        correlation_results = analyze_correlations(numerical_data, verbose)
+        results['correlation'] = correlation_results
+        
+        if verbose:
+            logger.warning("Performing dimensionality reduction...")
+        
+        dr_results = perform_dimensionality_reduction(scaled_data, numerical_columns, len(data), verbose)
+        results['dimensionality_reduction'] = dr_results
+        
+        if verbose:
+            logger.warning("Performing cluster analysis...")
+        
+        clustering_results = perform_cluster_analysis(scaled_data, numerical_data, len(data), verbose)
+        results['clustering'] = clustering_results
+
+        if target is not None:
+            if verbose:
+                logger.warning("Analyzing feature importance...")
+            
+            feature_importance_results = analyze_feature_importance(
+                data, target, numerical_columns, categorical_columns, target_is_numeric, verbose
+            )
+            results['feature_importance'] = feature_importance_results
+        
+        if verbose:
+            logger.warning("Detecting outliers...")
+        
+        outlier_results = detect_outliers(scaled_data, len(data), verbose)
+        results['outlier_detection'] = outlier_results
+        
+        if target is not None:
+            if verbose:
+                logger.warning("Performing statistical tests...")
+            
+            statistical_testing_results = perform_statistical_testing(
+                data, target, scaled_data, numerical_columns, categorical_columns, target_is_numeric, verbose
+            )
+            results['statistical_testing'] = statistical_testing_results
+        
+        if verbose:
+            logger.warning("Generating summary...")
+        
+        summary = generate_summary(results, df, verbose)
+        results['summary'] = summary
+        
+        if verbose:
+            logger.warning("Multivariate analysis completed successfully!")
+        
+        return results['summary']
+    
+    except Exception as e:
+        logger.error(f"Error in multivariate analysis: {str(e)}")
+        return {"error": str(e)}

src/app/pipelines/eda/tools/analysis_tools/univariate_analysis.py

@@ -0,0 +1,517 @@
+import pandas as pd
+import numpy as np
+from scipy import stats
+import missingno as msno
+from typing import Dict, List, Tuple, Optional, Union
+from agno.utils.log import logger
+import json
+
+def data_overview(data: pd.DataFrame, categorical_threshold) -> Dict:
+    """
+    Generate a high-level overview of the dataset structure.
+    
+    Returns:
+        A dictionary containing dataset dimensions and data types
+    """
+    logger.warn("Analyzing data overview...")
+    
+    rows, cols = data.shape
+    
+    dtypes = data.dtypes.value_counts().to_dict()
+    
+    total_observations = rows * cols
+    total_missing = data.isna().sum().sum()
+    completeness_ratio = (total_observations - total_missing) / total_observations
+    
+    feature_types = {}
+    for col in data.columns:
+        unique_count = data[col].nunique()
+        dtype = data[col].dtype
+        
+        if pd.api.types.is_numeric_dtype(dtype):
+            if unique_count <= categorical_threshold:
+                feature_types[col] = 'categorical (numeric)'
+            else:
+                feature_types[col] = 'numerical'
+        elif pd.api.types.is_string_dtype(dtype) or pd.api.types.is_object_dtype(dtype):
+            if unique_count <= categorical_threshold:
+                feature_types[col] = 'categorical'
+            else:
+                feature_types[col] = 'text'
+        elif pd.api.types.is_datetime64_dtype(dtype):
+            feature_types[col] = 'datetime'
+        elif pd.api.types.is_bool_dtype(dtype):
+            feature_types[col] = 'boolean'
+        else:
+            feature_types[col] = 'other'
+    
+    overview = {
+        'dimensions': {'rows': rows, 'columns': cols},
+        'data_types': dtypes,
+        'feature_types': feature_types,
+        'observations': {
+            'total': total_observations,
+            'missing': total_missing,
+            'completeness_ratio': completeness_ratio
+        }
+    }
+    
+    return overview
+
+def missing_values_analysis(data: pd.DataFrame) -> Dict:
+    """
+    Analyze missing values in the dataset.
+    
+    Returns:
+        A dictionary containing missing value statistics
+    """
+    logger.warn("Analyzing missing values...")
+    
+    missing_counts = data.isnull().sum()
+    missing_percentage = (missing_counts / len(data)) * 100
+    
+    columns_with_missing = missing_counts[missing_counts > 0].index.tolist()
+    
+    missing_correlation = None
+    if len(columns_with_missing) > 1:
+        missing_mask = data[columns_with_missing].isna()
+        missing_correlation = missing_mask.corr()
+    
+    missing_analysis = {
+        'missing_counts': missing_counts.to_dict(),
+        'missing_percentage': missing_percentage.to_dict(),
+        'columns_with_missing': columns_with_missing,
+        'total_missing_percentage': (missing_counts.sum() / (len(data) * len(data.columns))) * 100
+    }
+    
+    if missing_correlation is not None:
+        missing_analysis['missing_correlation'] = missing_correlation.to_dict()
+    
+    return missing_analysis
+
+def analyze_features(data: pd.DataFrame, results: dict) -> Dict:
+    """
+    Perform type-specific analysis for each feature.
+    
+    Returns:
+        A dictionary containing feature-level analysis results
+    """
+    logger.warn("Analyzing individual features...")
+    
+    feature_results = {}
+    
+    for column in data.columns:
+        feature_type = results['data_overview']['feature_types'][column]
+        
+        if 'numerical' in feature_type:
+            feature_results[column] = analyze_numerical(data, column)
+        elif 'categorical' in feature_type or feature_type == 'boolean':
+            feature_results[column] = analyze_categorical(data, column)
+        elif feature_type == 'datetime':
+            feature_results[column] = analyze_datetime(data, column)
+        elif feature_type == 'text':
+            feature_results[column] = analyze_text(data, column)
+        else:
+            feature_results[column] = {'type': feature_type, 'message': 'No specific analysis available for this type'}
+    
+    return feature_results
+
+def analyze_numerical(df: pd.DataFrame, column: str) -> Dict:
+        """
+        Perform comprehensive analysis for a numerical feature.
+        
+        Args:
+            column: The name of the column to analyze
+            
+        Returns:
+            A dictionary containing numerical statistics
+        """
+        data = df[column].dropna()
+        
+        if len(data) == 0:
+            return {'type': 'numerical', 'error': 'No non-null values found'}
+        
+        try:
+            stats_dict = {
+                'type': 'numerical',
+                'count': len(data),
+                'missing': df[column].isna().sum(),
+                'min': float(data.min()),
+                'max': float(data.max()),
+                'range': float(data.max() - data.min()),
+                'mean': float(data.mean()),
+                'median': float(data.median()),
+                'std': float(data.std()),
+                'variance': float(data.var()),
+                'quantiles': {
+                    '25%': float(data.quantile(0.25)),
+                    '50%': float(data.quantile(0.5)),
+                    '75%': float(data.quantile(0.75)),
+                    '90%': float(data.quantile(0.9)),
+                    '95%': float(data.quantile(0.95)),
+                    '99%': float(data.quantile(0.99))
+                }
+            }
+            
+            stats_dict['skewness'] = float(stats.skew(data))
+            stats_dict['kurtosis'] = float(stats.kurtosis(data))
+            
+            if len(data) >= 8:  # Minimum required for Shapiro-Wilk test
+                # Sample data if too large for tests
+                sample_data = data if len(data) < 5000 else data.sample(5000)
+                
+                try:
+                    shapiro_test = stats.shapiro(sample_data)
+                    stats_dict['normality_tests'] = {
+                        'shapiro_wilk': {
+                            'statistic': float(shapiro_test[0]),
+                            'p_value': float(shapiro_test[1]),
+                            'is_normal': shapiro_test[1] > 0.05
+                        }
+                    }
+                    
+                    if len(data) >= 20:  # Minimum for Anderson-Darling
+                        anderson_test = stats.anderson(sample_data, dist='norm')
+                        stats_dict['normality_tests']['anderson_darling'] = {
+                            'statistic': float(anderson_test.statistic),
+                            'critical_values': list(anderson_test.critical_values),
+                            'significance_levels': list(anderson_test.significance_level),
+                            'is_normal': anderson_test.statistic < anderson_test.critical_values[2]  # 5% significance
+                        }
+                except:
+                    stats_dict['normality_tests'] = {'error': 'Could not perform normality tests on this data'}
+            
+            q1 = data.quantile(0.25)
+            q3 = data.quantile(0.75)
+            iqr = q3 - q1
+            lower_bound = q1 - 1.5 * iqr
+            upper_bound = q3 + 1.5 * iqr
+            
+            outliers = data[(data < lower_bound) | (data > upper_bound)]
+            
+            stats_dict['outliers'] = {
+                'count': len(outliers),
+                'percentage': (len(outliers) / len(data)) * 100 if len(data) > 0 else 0,
+                'lower_bound': float(lower_bound),
+                'upper_bound': float(upper_bound)
+            }
+            
+            value_counts = data.value_counts().head(10).to_dict()
+            stats_dict['frequent_values'] = {str(k): int(v) for k, v in value_counts.items()}
+            
+            return stats_dict
+            
+        except Exception as e:
+            logger.warn(f"Error analyzing numerical column {column}: {str(e)}")
+            return {'type': 'numerical', 'error': str(e)}
+
+def analyze_categorical(df: pd.DataFrame, column: str) -> Dict:
+        """
+        Perform comprehensive analysis for a categorical feature.
+        
+        Args:
+            column: The name of the column to analyze
+            
+        Returns:
+            A dictionary containing categorical statistics
+        """
+        data = df[column].dropna()
+        
+        # Skip if no data available
+        if len(data) == 0:
+            return {'type': 'categorical', 'error': 'No non-null values found'}
+        
+        try:
+            # Basic statistics
+            value_counts = data.value_counts()
+            value_percentages = data.value_counts(normalize=True) * 100
+            
+            stats_dict = {
+                'type': 'categorical',
+                'count': len(data),
+                'missing': df[column].isna().sum(),
+                'unique_values': data.nunique(),
+                'mode': str(data.mode().iloc[0]) if not data.mode().empty else None,
+                'entropy': float(stats.entropy(value_counts / len(data))) if len(value_counts) > 1 else 0
+            }
+            
+            # Category frequencies
+            max_categories = 30  # Limit to top 30 categories to prevent huge outputs
+            if len(value_counts) <= max_categories:
+                stats_dict['categories'] = {
+                    str(category): {
+                        'count': int(count),
+                        'percentage': float(value_percentages[category])
+                    } for category, count in value_counts.items()
+                }
+            else:
+                # Include top categories and group the rest as "other"
+                top_categories = value_counts.head(max_categories)
+                other_count = value_counts.iloc[max_categories:].sum()
+                other_percentage = value_percentages.iloc[max_categories:].sum()
+                
+                stats_dict['categories'] = {
+                    str(category): {
+                        'count': int(count),
+                        'percentage': float(value_percentages[category])
+                    } for category, count in top_categories.items()
+                }
+                
+                stats_dict['categories']['other'] = {
+                    'count': int(other_count),
+                    'percentage': float(other_percentage)
+                }
+                
+                stats_dict['note'] = f"Showing top {max_categories} of {len(value_counts)} categories. Remaining grouped as 'other'."
+            
+            return stats_dict
+            
+        except Exception as e:
+            logger.warn(f"Error analyzing categorical column {column}: {str(e)}")
+            return {'type': 'categorical', 'error': str(e)}
+
+def analyze_datetime(df: pd.DataFrame, column: str) -> Dict:
+        """
+        Perform comprehensive analysis for a datetime feature.
+        
+        Args:
+            column: The name of the column to analyze
+            
+        Returns:
+            A dictionary containing datetime statistics
+        """
+        data = df[column].dropna()
+        
+        # Skip if no data available
+        if len(data) == 0:
+            return {'type': 'datetime', 'error': 'No non-null values found'}
+        
+        try:
+            stats_dict = {
+                'type': 'datetime',
+                'count': len(data),
+                'missing': df[column].isna().sum(),
+                'min_date': str(data.min()),
+                'max_date': str(data.max()),
+                'range_days': (data.max() - data.min()).days if hasattr((data.max() - data.min()), 'days') else None
+            }
+            
+            # Distribution by year, month, day of week if there are enough dates
+            if len(data) >= 10:
+                try:
+                    stats_dict['year_counts'] = data.dt.year.value_counts().sort_index().to_dict()
+                    stats_dict['month_counts'] = data.dt.month.value_counts().sort_index().to_dict()
+                    stats_dict['day_of_week_counts'] = data.dt.dayofweek.value_counts().sort_index().to_dict()
+                    stats_dict['hour_counts'] = data.dt.hour.value_counts().sort_index().to_dict() if hasattr(data.dt, 'hour') else None
+                except:
+                    # Some datetime objects might not support certain attributes
+                    pass
+            
+            return stats_dict
+            
+        except Exception as e:
+            logger.warn(f"Error analyzing datetime column {column}: {str(e)}")
+            return {'type': 'datetime', 'error': str(e)}
+
+def analyze_text(df: pd.DataFrame, column: str) -> Dict:
+        """
+        Perform basic analysis for a text feature.
+        
+        Args:
+            column: The name of the column to analyze
+            
+        Returns:
+            A dictionary containing text statistics
+        """
+        data = df[column].dropna().astype(str)
+        
+        # Skip if no data available
+        if len(data) == 0:
+            return {'type': 'text', 'error': 'No non-null values found'}
+        
+        try:
+            text_lengths = data.str.len()
+            word_counts = data.str.split().str.len()
+            
+            stats_dict = {
+                'type': 'text',
+                'count': len(data),
+                'missing': df[column].isna().sum(),
+                'unique_values': data.nunique(),
+                'text_length': {
+                    'min': int(text_lengths.min()),
+                    'max': int(text_lengths.max()),
+                    'mean': float(text_lengths.mean()),
+                    'median': float(text_lengths.median())
+                }
+            }
+            
+            # Word count statistics if available
+            if not word_counts.isna().all():
+                stats_dict['word_count'] = {
+                    'min': int(word_counts.min()),
+                    'max': int(word_counts.max()),
+                    'mean': float(word_counts.mean()),
+                    'median': float(word_counts.median())
+                }
+            
+            # Sample values (first few characters)
+            max_samples = 5
+            max_length = 100  # Show only first 100 chars for each sample
+            
+            if len(data) <= max_samples:
+                samples = data.tolist()
+            else:
+                samples = data.sample(max_samples).tolist()
+            
+            stats_dict['samples'] = [f"{s[:max_length]}{'...' if len(s) > max_length else ''}" for s in samples]
+            
+            return stats_dict
+            
+        except Exception as e:
+            logger.warning(f"Error analyzing text column {column}: {str(e)}")
+            return {'type': 'text', 'error': str(e)}
+
+def generate_summary_report(results: dict) -> Dict:
+        """
+        Generate a summary of the univariate analysis results.
+        
+        Returns:
+            A dictionary containing the summary report
+        """
+        logger.info("Generating summary report...")
+        
+        # Extract key insights
+        summary = {
+            'dataset_summary': {
+                'dimensions': results['data_overview']['dimensions'],
+                'completeness': f"{results['data_overview']['observations']['completeness_ratio']*100:.2f}%"
+            },
+            'feature_types': {
+                'numerical': [],
+                'categorical': [],
+                'datetime': [],
+                'text': [],
+                'other': []
+            },
+            'missing_values': {
+                'total_missing_percentage': f"{results['missing_values']['total_missing_percentage']:.2f}%",
+                'features_with_high_missingness': []
+            },
+            'numerical_features': {
+                'highly_skewed': [],
+                'potentially_non_normal': []
+            },
+            'categorical_features': {
+                'high_cardinality': [],
+                'binary': []
+            }
+        }
+        
+        # Categorize features by type
+        for col, feat_type in results['data_overview']['feature_types'].items():
+            if 'numerical' in feat_type:
+                summary['feature_types']['numerical'].append(col)
+            elif 'categorical' in feat_type or feat_type == 'boolean':
+                summary['feature_types']['categorical'].append(col)
+            elif feat_type == 'datetime':
+                summary['feature_types']['datetime'].append(col)
+            elif feat_type == 'text':
+                summary['feature_types']['text'].append(col)
+            else:
+                summary['feature_types']['other'].append(col)
+        
+        # Features with high missingness (>10%)
+        for col, miss_pct in results['missing_values']['missing_percentage'].items():
+            if miss_pct > 10:
+                summary['missing_values']['features_with_high_missingness'].append({
+                    'feature': col,
+                    'missing_percentage': f"{miss_pct:.2f}%"
+                })
+        
+        # Analyze numerical features
+        for col in summary['feature_types']['numerical']:
+            if col in results['feature_analysis']:
+                analysis = results['feature_analysis'][col]
+                
+                if 'skewness' in analysis and abs(analysis['skewness']) > 1:
+                    summary['numerical_features']['highly_skewed'].append({
+                        'feature': col,
+                        'skewness': analysis['skewness']
+                    })
+                
+                if 'normality_tests' in analysis and isinstance(analysis['normality_tests'], dict):
+                    for test, res in analysis['normality_tests'].items():
+                        if isinstance(res, dict) and 'is_normal' in res and not res['is_normal']:
+                            summary['numerical_features']['potentially_non_normal'].append({
+                                'feature': col,
+                                'test': test
+                            })
+                            break
+        
+        # Analyze categorical features
+        for col in summary['feature_types']['categorical']:
+            if col in results['feature_analysis']:
+                analysis = results['feature_analysis'][col]
+                
+                if 'unique_values' in analysis:
+                    if analysis['unique_values'] > 20:
+                        summary['categorical_features']['high_cardinality'].append({
+                            'feature': col,
+                            'unique_values': analysis['unique_values']
+                        })
+                    elif analysis['unique_values'] == 2:
+                        summary['categorical_features']['binary'].append(col)
+        
+        return summary
+
+def univariate_analysis(data_path: str, categorical_threshold: int = 10):
+    """
+    Perform a  comprehensive univariate analysis on a dataset.
+    
+    Args:
+        data_path: Path to a data file or a pandas DataFrame
+        categorical_threshold: Maximum unique values to consider a feature categorical
+    Returns:
+        A dictionary containing all analysis results
+    """
+    try:
+        if isinstance(data_path, str):
+            data = pd.read_csv(data_path)
+        else:
+            logger.error(f"Unsupported file format: {data_path}")
+            return {"error": f"Unsupported file format: {data_path}"}
+
+        if not isinstance(data, pd.DataFrame):
+            logger.error(f"Input is not a valid pandas DataFrame")
+            return {"error": "Input is not a valid pandas DataFrame"}
+
+        logger.warn("Staring Analysis.....")
+
+        copy_data = data.copy()
+
+        results = {}
+
+        results['data_overview'] = data_overview(
+            data=copy_data,
+            categorical_threshold=categorical_threshold
+        )
+
+        results['missing_values'] = missing_values_analysis(
+            data=copy_data
+        )
+
+        results['feature_analysis'] = analyze_features(
+            data=copy_data,
+            results = results
+        )
+        
+        analysis_summary = generate_summary_report(results)
+
+        return json.dumps(analysis_summary)
+        
+    except Exception as e:
+        logger.error(f"Error in analyze_dataset: {str(e)}")
+        return {"error": str(e)}    
+
+

src/app/pipelines/eda/tools/data_cleaning_tools/__init__.py

@@ -0,0 +1,2 @@
+from .handle_missing_values import handle_missing_values
+from .handle_outliers import handle_outliers

src/app/pipelines/eda/tools/data_cleaning_tools/handle_missing_values.py

@@ -0,0 +1,64 @@
+import pandas as pd
+from agno.utils.log import logger
+import json
+
+def handle_missing_values(file_path: str) -> list:
+        """
+        Loads a CSV file using pandas, handles missing values using statistical methods,
+        and saves the processed file to storage "outputs/".
+
+        Args:
+            file_path (str): Path to the CSV file.
+        Returns:
+            Status of the entire process
+        """
+        try:
+            df = pd.read_csv(file_path)
+            logger.info(f"Started Missing Values Handler. CSV Loaded with shape: {df.shape} ")
+
+            missing_before = df.isnull().sum().to_dict()
+
+            result_df = df.copy()
+        
+            numeric_cols = result_df.select_dtypes(include=['number']).columns.tolist()
+            categorical_cols = result_df.select_dtypes(include=['category']).columns.tolist()
+            text_cols = result_df.select_dtypes(include=['object']).columns.tolist()
+            
+            
+            for col in numeric_cols:
+                if result_df[col].isnull().any():
+                    median_value = result_df[col].median()
+                    result_df[col] = result_df[col].fillna(median_value)
+                    logger.info(f"Filled {col} missing values with median: {median_value}")
+            
+            for col in categorical_cols:
+                if result_df[col].isnull().any():
+                    if not result_df[col].mode().empty:
+                        mode_value = result_df[col].mode()[0]
+                        result_df[col] = result_df[col].fillna(mode_value)
+                        logger.info(f"Filled {col} missing values with mode: {mode_value}")
+                    else:
+                        logger.warning(f"Column {col} has no mode. Missing values remain.")
+            
+            for col in text_cols:
+                if result_df[col].isnull().any():
+                    if not result_df[col].mode().empty:
+                        mode_value = result_df[col].mode()[0]
+                        result_df[col] = result_df[col].fillna(mode_value)
+                        logger.info(f"Filled text column {col} missing values with most frequent value")
+                    else:
+                        result_df[col] = result_df[col].fillna("")
+                        logger.info(f"Filled text column {col} missing values with empty string")
+
+            output_dir = "src/core/cache/dataset_logs"
+
+            missing_after = result_df.isnull().sum().to_dict()
+            result_df.to_csv(f"{output_dir}/handle_missing_values_output.csv", index=False)
+
+            logger.info(f"CSV output stored with shape: {result_df.shape} ")
+            
+            return json.dumps({'status': 'success'})
+
+        except Exception as e:
+            logger.warning(f"Failed to process CSV file: {e}")
+            return str(e)

src/app/pipelines/eda/tools/data_cleaning_tools/handle_outliers.py

@@ -0,0 +1,83 @@
+import pandas as pd
+import numpy as np
+from agno.tools import Toolkit
+from agno.utils.log import logger
+import json
+
+def _detect_outliers(df: pd.DataFrame, threshold: float = 3.5, remove_outliers: bool = True) -> pd.DataFrame:
+        """
+        Detect and handle outliers in numerical columns using the Modified Z-Score method.
+
+        Args:
+            df (pd.DataFrame): Input dataframe.
+            threshold (float): The threshold for detecting outliers (default is 3.5).
+            remove_outliers (bool): Whether to remove outliers (True) or just mark them (False).
+
+        Returns:
+            pd.DataFrame: DataFrame with outliers handled.
+        """
+        result_df = df.copy()
+        numeric_cols = result_df.select_dtypes(include=[np.number]).columns.tolist()
+
+        if not numeric_cols:
+            logger.warning("No numerical columns found. Skipping outlier handling.")
+            return result_df
+
+        outlier_flags = pd.DataFrame(index=result_df.index)
+
+        for col in numeric_cols:
+            median = result_df[col].median()
+            mad = np.median(np.abs(result_df[col] - median))
+
+            if mad == 0:  # Prevent division by zero
+                continue
+
+            # Compute Modified Z-Score
+            mod_z_score = 0.6745 * (result_df[col] - median) / mad
+
+            # Mark outliers
+            outlier_flags[col + '_outlier'] = np.abs(mod_z_score) > threshold
+
+        result_df['is_outlier'] = outlier_flags.any(axis=1)
+
+        if remove_outliers:
+            cleaned_df = result_df[~result_df['is_outlier']].drop(columns=['is_outlier'])
+            logger.info(f"Removed {result_df['is_outlier'].sum()} outliers.")
+        else:
+            cleaned_df = result_df
+
+        return cleaned_df
+
+def handle_outliers(file_path: str, threshold: float = 3.5, remove_outliers: bool = True) -> dict:
+        """
+        Loads a CSV file, detects/removes outliers using the Modified Z-Score method, and saves the cleaned data to /outputs
+
+        Args:
+            file_path (str): Path to the CSV file.
+            threshold (float): The threshold for detecting outliers.
+            remove_outliers (bool): Whether to remove outliers or just mark them.
+
+        Returns:
+            dict: Process status, outlier statistics, and output file path.
+        """
+        try:
+            df = pd.read_csv(file_path)
+            logger.warn(f"Started Outlier Detection. CSV Loaded with shape: {df.shape}")
+
+            outliers_before = df.isnull().sum().to_dict()
+            df = _detect_outliers(df, threshold, remove_outliers)
+
+            outliers_after = df.isnull().sum().to_dict()
+
+            output_dir = "src/core/cache/dataset_logs"
+
+            output_path = f"{output_dir}/outlier_detection_output.csv"
+            df.to_csv(output_path, index=False)
+
+            logger.info(f"CSV output stored at {output_path} with shape: {df.shape}")
+
+            return json.dumps(f"Total outliers before: {outliers_before}. Total outliers after: {outliers_after}. The threshold used for dealing with outliers is {threshold}. The file is stored in path: '{output_path}' ")
+
+        except Exception as e:
+            logger.warning(f"Failed to process CSV file: {e}")
+            return {"status": "Error", "message": str(e)}

src/app/pipelines/eda/tools/lib.py

@@ -0,0 +1,59 @@
+from .analysis_tools import univariate_analysis, bivariate_analysis, multivariate_analysis
+from .data_cleaning_tools import handle_outliers, handle_missing_values
+
+tool_library = {
+    # "HandleMissingValues": {
+    #     "name": "Missing Values Handler",
+    #     "function": handle_missing_values,
+    #     "metadata": '''
+    #     1. Fills missing values with the median of each column.
+    #     2. Fills missing values with the mode, if available; otherwise, logs a warning.
+    #     3. Fills missing values with the most frequent value or an empty string if mode is unavailable.
+    #                 ''',
+    # },
+    "handle_outliers": {
+        "name": "Outlier Handler",
+        "function": handle_outliers,
+        "metadata": '''
+        1. Uses median and MAD (Median Absolute Deviation) to detect outliers.
+        2. Identifies extreme values based on a set threshold and either excludes them from the dataset or keeps them marked for reference.
+                    ''',
+    },
+    'univariate_analysis': {
+        "name": "Univariate Analysis",
+        "function": univariate_analysis,
+        "metadata": '''
+        1. Provides a high-level summary of dataset structure, data types, and missing value statistics.
+        2. Analyzes missing values, their distribution, and correlation between missing columns.
+        3. Performs feature-specific analysis based on detected data types
+        4. Computes descriptive statistics, normality tests, and outlier detection for numerical columns.
+        5. Analyzes categorical distributions, entropy, and category frequencies with top values.
+        6. (Truncated but likely) extracts patterns, ranges, and trends from datetime columns.
+                    ''',
+    },
+    'bivariate_analysis': {
+        "name": "Bivariate Analysis",
+        "function": bivariate_analysis,
+        "metadata": '''
+        1. Uses Pearson, Spearman, and Kendall correlations for numerical variables, chi-square/Cramér’s V for categorical associations, and statistical tests like ANOVA for numerical vs. categorical analysis. Identifies best-fit relationships (linear, polynomial, etc.) for numerical pairs.
+        2. Provides a detailed bivariate analysis of all variable pairs in a dataframe, summarizing key correlations, associations, and insights. Optionally generates and saves visualizations like scatterplots and heatmaps.
+        3. Uses Chi-square tests and Cramer's V to assess categorical feature associations, calculates Phi coefficient for 2x2 tables, and computes Goodman & Kruskal’s Lambda for predictive strength.
+        4. Identifies statistically significant relationships between categorical variables, ranks them by strength, and optionally visualizes contingency tables as heatmaps.
+        5. The function performs ANOVA (One-Way & Welch’s ANOVA), Point-Biserial Correlation (for binary categories), and Levene’s test to analyze relationships between numerical and categorical features, calculating effect sizes (eta-squared, omega-squared) for significance testing.
+                    ''',
+    },
+    'multivariate_analysis': {
+        "name": "Multivariate Analysis",
+        "function": multivariate_analysis,
+        "metadata": '''
+        1. Calculates the pairwise correlation coefficients between all numerical columns in a given DataFrame, generating a correlation matrix.
+        2. It identifies pairs of numerical features with absolute correlation values exceeding a threshold of 0.7, indicating strong linear relationships.
+        3. Calculates the Variance Inflation Factor (VIF) for each numerical feature to detect multicollinearity, flagging features with VIF values greater than 10 as potential issues.
+        4. Uses PCA, Factor Analysis, t-SNE, and MDS. Identifies principal components or latent factors, aiming for 80% variance retention in PCA.
+        5. Finds optimal clusters using silhouette score, evaluates cluster quality. Density-based clustering for smaller datasets (<=5000 rows), identifies noise. Fits Gaussian mixture models, evaluates model fit.
+        6. Statistical tests and mutual information to rank individual feature relevance. Random Forest models to determine feature contribution to prediction.  Iterative feature removal to select top features (max 10).
+        7. Detects outliers by isolating them in random partitions, using a contamination rate of 5%. Identifies local density deviations for smaller datasets (<= 5000 rows), also using a 5percent contamination rate and 20 neighbors. Provides the number and percentage of detected outliers for each method.
+        8. MANOVA: Tests mean differences across categorical target groups for multiple numerical features. LDA: Dimensionality reduction and classification for categorical targets.
+        '''
+    }
+}

src/app/pipelines/modules/__init__.py

@@ -0,0 +1,4 @@
+from .data_statistics import DataStatisticsWorkflow
+from .data_quality_assessment import DataQualityAssessmentWorkflow
+from .data_understanding_context import DataUnderstandingContextWorkflow
+from .univariate_analysis import UnivariateAnalysisWorkflow

src/app/pipelines/modules/data_quality_assessment.py

@@ -0,0 +1,1657 @@
+import re
+import os
+import ast
+import json
+import difflib
+import numpy as np
+import pandas as pd
+from scipy import stats
+from typing import Union
+from collections import Counter
+from src.core.utils import logger
+from scipy.spatial.distance import pdist
+from scipy.stats import chi2_contingency
+from agno.agent import Agent, RunResponse
+from agno.models.openai import OpenAIChat
+from src.core.utils import KnowledgeBaseClass
+from scipy.cluster.hierarchy import linkage, fcluster
+from sklearn.metrics.pairwise import cosine_similarity
+from typing import Dict, List, Union, Tuple, Any, Optional
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+class DataQualityAssessmentWorkflow:
+    def __init__(
+        self, data_source: str,
+        llm_choice: str,
+        ml_task: str
+    ) -> None:
+        ''''''
+        self.data = None
+        self.data_source = data_source
+        self.llm_choice = llm_choice
+        self.ml_task = ml_task
+        self.llm = OpenAIChat(id=llm_choice, api_key=os.getenv('OPENAI_API_KEY'))
+        self.writer: Agent = Agent(
+            model=self.llm,
+            instructions=[
+                "You will be provided with lots of structured outputs. Your work is to display this"
+                "in a nicely formatted manner. You must analayze the results and output a comprehensive and insightful report"
+            ],
+            markdown=True,
+        )
+        _ = self.load_data(data_source=data_source)
+
+    def load_data(self, data_source: str) -> Union[None, bool]:
+        '''Load CSV into dataframe'''
+        try:
+            self.data = pd.read_csv(data_source)
+            return True
+        except Exception as e:
+            logger.error(
+                f"Failed to read the file from the data source with error: {e}", log_type="data_quality_assessment", console=True)
+            return False
+    
+    def analyze_missing_data(self, verbose=False) -> Dict:
+        """Comprehensive analysis of missing data patterns in a CSV dataset"""
+        results = {
+            "status": "success",
+            "data_loaded": False,
+            "data_shape": None,
+            "missing_rates": None,
+            "little_mcar_test": None,
+            "mar_correlations": None,
+            "mutual_missingness": None,
+            "segment_analysis": None,
+            "summary": None,
+            "errors": []
+        }
+        
+        try:
+            df = self.data
+            results["data_loaded"] = True
+            results["data_shape"] = df.shape
+        except Exception as e:
+            results["status"] = "failed"
+            results["errors"].append(f"Failed to load data: {str(e)}")
+            return results
+        
+        try:
+            results["missing_rates"] = self.analyze_missing_rates(df, verbose=verbose)
+        except Exception as e:
+            results["errors"].append(f"Failed to analyze missing rates: {str(e)}")
+            logger.error(f"Failed to analyze missing rates: {str(e)}", log_type='data_quality_assessment', console=verbose)
+
+        try:
+            results["little_mcar_test"] = self.littles_mcar_test(df, verbose=verbose)
+        except Exception as e:
+            results["errors"].append(f"Failed to perform Little's MCAR test: {str(e)}")
+            logger.error(f"Failed to perform Little's MCAR test: {str(e)}", log_type='data_quality_assessment', console=verbose)
+        
+        try:
+            results["mar_correlations"] = self.check_mar_correlations(df, verbose=verbose)
+        except Exception as e:
+            results["errors"].append(f"Failed to perform MAR correlation check: {str(e)}")
+            logger.error(f"Failed to perform MAR correlation check: {str(e)}", log_type='data_quality_assessment', console=verbose)
+        
+        try:
+            results["mutual_missingness"] = self.analyze_mutual_missingness(df, verbose=verbose)
+        except Exception as e:
+            results["errors"].append(f"Failed to analyze mutual missingness: {str(e)}")
+            logger.error(f"Failed to analyze mutual missingness: {str(e)}", log_type='data_quality_assessment', console=verbose)
+        
+        try:
+            results["segment_analysis"] = self.segment_based_analysis(df, verbose=verbose)
+        except Exception as e:
+            results["errors"].append(f"Failed to perform segment-based analysis: {str(e)}")
+            logger.error(f"Failed to perform segment-based analysis: {str(e)}", log_type='data_quality_assessment', console=verbose)
+        
+        results["summary"] = self.generate_summary(results, verbose=verbose)
+        
+        return results
+    
+    def analyze_missing_rates(self, df: pd.DataFrame, verbose=False) -> Dict:
+        """Calculate per-variable/column missing rates"""
+        try:
+            missing_count = df.isnull().sum()
+            missing_percentage = (missing_count / len(df)) * 100
+            
+            missing_info = pd.DataFrame({
+                'Column': missing_count.index,
+                'Missing Count': missing_count.values,
+                'Missing Percentage': missing_percentage.values
+            })
+            
+            missing_info = missing_info.sort_values('Missing Percentage', ascending=False)
+            
+            total_cells = df.shape[0] * df.shape[1]
+            total_missing = df.isnull().sum().sum()
+            overall_missing_percentage = (total_missing / total_cells) * 100
+            
+            result = {
+                'per_column': missing_info.to_dict('records'),
+                'overall': {
+                    'total_cells': total_cells,
+                    'total_missing': total_missing,
+                    'overall_missing_percentage': overall_missing_percentage
+                }
+            }
+            
+            return result
+        except Exception as e:
+            logger.error(f"Error in analyze_missing_rates: {str(e)}", log_type="data_quality_assessment", console=verbose)
+            return {'error': str(e)}
+        
+    def littles_mcar_test(self, df: pd.DataFrame, verbose=False) -> Dict:
+        """Perform Little's MCAR test to check if data is Missing Completely At Random"""
+        try:
+            numeric_df = df.select_dtypes(include=[np.number])
+            if numeric_df.empty:
+                return {
+                    'status': 'skipped',
+                    'reason': 'No numeric columns available for Little\'s MCAR test'
+                }
+            
+            if numeric_df.shape[0] < 3 or numeric_df.shape[1] < 2:
+                return {
+                    'status': 'skipped',
+                    'reason': 'Not enough data for meaningful Little\'s MCAR test'
+                }
+            
+            results = []
+            
+            for col in numeric_df.columns:
+                try:
+                    missing_indicator = numeric_df[col].isnull()
+                    
+                    if missing_indicator.sum() == 0 or missing_indicator.sum() == len(missing_indicator):
+                        continue
+                    
+                    for other_col in numeric_df.columns:
+                        if col == other_col:
+                            continue
+                        
+                        if numeric_df[other_col].isnull().sum() == len(numeric_df):
+                            continue
+                        
+                        not_missing_values = numeric_df.loc[~missing_indicator, other_col].dropna()
+                        missing_values = numeric_df.loc[missing_indicator, other_col].dropna()
+                        
+                        if len(not_missing_values) > 5 and len(missing_values) > 5:
+                            try:
+                                not_missing_mean = not_missing_values.mean()
+                                missing_mean = missing_values.mean()
+                                
+                                t_stat, p_value = stats.ttest_ind(
+                                    not_missing_values, 
+                                    missing_values,
+                                    equal_var=False,
+                                    nan_policy='omit'
+                                )
+                                
+                                results.append({
+                                    'reference_col': col,
+                                    'test_col': other_col,
+                                    'not_missing_mean': not_missing_mean,
+                                    'missing_mean': missing_mean,
+                                    'difference': abs(not_missing_mean - missing_mean),
+                                    't_statistic': t_stat,
+                                    'p_value': p_value,
+                                    'significant': p_value < 0.05
+                                })
+                            except Exception:
+                                pass
+                except:
+                    continue
+            significant_tests = [test for test in results if test.get('significant', False)]
+            
+            if not results:
+                is_mcar = None
+                evidence_strength = None
+            else:
+                prop_significant = len(significant_tests) / len(results)
+                
+                is_mcar = prop_significant <= 0.05
+                
+                if prop_significant == 0:
+                    evidence_strength = "Strong evidence for MCAR"
+                elif prop_significant <= 0.05:
+                    evidence_strength = "Moderate evidence for MCAR"
+                elif prop_significant <= 0.10:
+                    evidence_strength = "Weak evidence against MCAR"
+                else:
+                    evidence_strength = "Strong evidence against MCAR"
+            
+            return {
+                'status': 'completed',
+                'is_mcar': is_mcar,
+                'tests_conducted': len(results),
+                'significant_tests': len(significant_tests),
+                'proportion_significant': len(significant_tests) / len(results) if results else None,
+                'evidence_strength': evidence_strength,
+                'test_details': results[:10] 
+            }
+        except Exception as e:
+            logger.error(f"Error in littles_mcar_test: {str(e)}", log_type="data_quality_assessment", console=verbose)
+            return {'status': 'error', 'error': str(e)}
+        
+    def check_mar_correlations(self, df: pd.DataFrame, verbose=False) -> Dict:
+        """Check if missingness in one variable depends on observed values in others (MAR)"""
+        try:
+            results = {}
+            
+            for col in df.columns:
+                try:
+                    missing_count = df[col].isnull().sum()
+                    if missing_count == 0 or missing_count == len(df):
+                        continue  
+                        
+                    missingness = df[col].isnull().astype(int)
+                    
+                    col_results = {}
+                    
+                    for other_col in df.columns:
+                        if col == other_col or df[other_col].isnull().sum() == len(df):
+                            continue
+                        
+                        if pd.api.types.is_numeric_dtype(df[other_col]):
+                            try:
+                                present_values = df.loc[~df[col].isnull(), other_col].dropna()
+                                missing_values = df.loc[df[col].isnull(), other_col].dropna()
+                                
+                                if len(present_values) > 5 and len(missing_values) > 5:
+                                    present_mean = present_values.mean()
+                                    missing_mean = missing_values.mean()
+                                    
+                                    t_stat, p_value = stats.ttest_ind(
+                                        present_values, 
+                                        missing_values,
+                                        equal_var=False,
+                                        nan_policy='omit'
+                                    )
+                                    
+                                    if p_value < 0.05:
+                                        col_results[other_col] = {
+                                            'test': 't-test',
+                                            'present_mean': present_mean,
+                                            'missing_mean': missing_mean,
+                                            'mean_difference': missing_mean - present_mean,
+                                            't_statistic': t_stat,
+                                            'p_value': p_value,
+                                            'significant': True
+                                        }
+                            except Exception as e:
+                                pass
+                        
+                        elif pd.api.types.is_object_dtype(df[other_col]) or pd.api.types.is_categorical_dtype(df[other_col]):
+                            try:
+                                value_counts = df[other_col].value_counts()
+                                threshold = max(5, len(df) * 0.01)  
+                                
+                                temp_df = df.copy()
+                                
+                                low_freq_cats = value_counts[value_counts < threshold].index.tolist()
+                                if low_freq_cats:
+                                    temp_df[other_col] = temp_df[other_col].apply(
+                                        lambda x: 'Other' if x in low_freq_cats else x
+                                    )
+                                
+                                contingency = pd.crosstab(
+                                    temp_df[col].isnull(),
+                                    temp_df[other_col].fillna('Missing')
+                                )
+                                
+                                if (contingency.shape[0] > 1 and contingency.shape[1] > 1 and 
+                                    (contingency < 5).sum().sum() <= contingency.size * 0.2): 
+                                    
+                                    chi2, p, dof, expected = chi2_contingency(contingency)
+                                    
+                                    if p < 0.05:
+                                        col_results[other_col] = {
+                                            'test': 'chi-square',
+                                            'chi2': chi2,
+                                            'p_value': p,
+                                            'dof': dof,
+                                            'significant': True
+                                        }
+                            except Exception as e:
+                                pass
+                    
+                    if col_results:
+                        results[col] = col_results
+                except:
+                    continue
+            
+            is_mar = len(results) > 0
+            
+            return {
+                'is_mar': is_mar,
+                'details': results,
+                'columns_with_mar_evidence': list(results.keys())
+            }
+        except Exception as e:
+            logger.error(f"Error in check_mar_correlations: {str(e)}", log_type="data_quality_assessment", console=verbose)
+            return {'error': str(e)}
+        
+    def analyze_mutual_missingness(self, df: pd.DataFrame, verbose=False) -> Dict:
+        """Check for mutual missingness - analyze if certain features tend to be missing together"""
+        try:
+            binary_missing = df.isnull().astype(int)
+            
+            if binary_missing.sum().sum() == 0:
+                return {
+                    'status': 'skipped',
+                    'reason': 'No missing values found in dataset'
+                }
+            
+            missingness_correlation = binary_missing.corr()
+            
+            strong_correlations = []
+            
+            for i in range(len(missingness_correlation.columns)):
+                try: 
+                    for j in range(i+1, len(missingness_correlation.columns)):
+                        col1 = missingness_correlation.columns[i]
+                        col2 = missingness_correlation.columns[j]
+                        corr_value = missingness_correlation.iloc[i, j]
+                        
+                        if abs(corr_value) > 0.5:
+                            contingency = pd.crosstab(
+                                binary_missing[col1],
+                                binary_missing[col2]
+                            )
+                            
+                            try:
+                                if contingency.values.min() < 5:
+                                    from scipy.stats import fisher_exact
+                                    _, p_value = fisher_exact(contingency)
+                                else:
+                                    _, p_value, _, _ = chi2_contingency(contingency)
+                                
+                                if p_value < 0.05:
+                                    strong_correlations.append({
+                                        'column1': col1,
+                                        'column2': col2,
+                                        'correlation': corr_value,
+                                        'p_value': p_value,
+                                        'significant': True
+                                    })
+                            except Exception:
+                                strong_correlations.append({
+                                    'column1': col1,
+                                    'column2': col2,
+                                    'correlation': corr_value,
+                                    'significant': None
+                                })
+                except:
+                    continue
+            
+            co_occurrence = {}
+            for i in range(len(df.columns)):
+                try: 
+                    for j in range(i+1, len(df.columns)):
+                        col1 = df.columns[i]
+                        col2 = df.columns[j]
+                        
+                        both_missing = (df[col1].isnull() & df[col2].isnull()).sum()
+                        
+                        if both_missing > 0:
+                            col1_missing = df[col1].isnull().sum()
+                            col2_missing = df[col2].isnull().sum()
+                            
+                            union_missing = (df[col1].isnull() | df[col2].isnull()).sum()
+                            jaccard = both_missing / union_missing if union_missing > 0 else 0
+                            
+                            co_occurrence[(col1, col2)] = {
+                                'both_missing': both_missing,
+                                'col1_missing': col1_missing,
+                                'col2_missing': col2_missing,
+                                'co_occurrence_ratio': both_missing / min(col1_missing, col2_missing) if min(col1_missing, col2_missing) > 0 else 0,
+                                'jaccard_coefficient': jaccard
+                            }
+                except:
+                    continue
+            
+            co_occurrence_list = [
+                {
+                    'column1': cols[0], 
+                    'column2': cols[1],
+                    'both_missing_count': data['both_missing'],
+                    'co_occurrence_ratio': data['co_occurrence_ratio'],
+                    'jaccard_coefficient': data['jaccard_coefficient']
+                }
+                for cols, data in co_occurrence.items()
+            ]
+            co_occurrence_list.sort(key=lambda x: x['co_occurrence_ratio'], reverse=True)
+            
+            missingness_clusters = None
+            try:
+                if len(binary_missing.columns) > 1: 
+
+                    
+                    cols_with_missing = [col for col in binary_missing.columns if binary_missing[col].sum() > 0]
+                    if len(cols_with_missing) > 1:
+                        missing_data = binary_missing[cols_with_missing].T  
+                        
+                        dist_matrix = pdist(missing_data, metric='correlation')
+                        
+                        linkage_matrix = linkage(dist_matrix, method='average')
+                        
+                        clusters = fcluster(linkage_matrix, t=0.5, criterion='distance')
+                        
+                        missingness_clusters = {}
+                        for i, col in enumerate(cols_with_missing):
+                            cluster_id = clusters[i]
+                            if cluster_id not in missingness_clusters:
+                                missingness_clusters[cluster_id] = []
+                            missingness_clusters[cluster_id].append(col)
+                        
+                        missingness_clusters = {k: v for k, v in missingness_clusters.items() if len(v) > 1}
+            except Exception as e:
+                logger.info(f"Warning: Clustering of missingness patterns failed: {str(e)}", log_type="data_quality_assessment", console=verbose)
+            
+            return {
+                'status': 'completed',
+                'strong_correlations': strong_correlations,
+                'co_occurrence': co_occurrence_list[:15], 
+                'missingness_clusters': missingness_clusters
+            }
+        except Exception as e:
+            logger.error(f"Error in analyze_mutual_missingness: {str(e)}", log_type='data_quality_assessment', console=verbose)
+            return {'status': 'error', 'error': str(e)}
+        
+    def segment_based_analysis(self, df: pd.DataFrame, verbose=False) -> Dict:
+        """Analyze missingness grouped by categories"""
+        try:
+            results = {}
+            
+            categorical_cols = [
+                col for col in df.columns 
+                if pd.api.types.is_object_dtype(df[col]) or 
+                pd.api.types.is_categorical_dtype(df[col]) or
+                (pd.api.types.is_numeric_dtype(df[col]) and df[col].nunique() < 10)
+            ]
+            
+            if not categorical_cols:
+                return {
+                    'status': 'skipped',
+                    'reason': 'No suitable categorical columns found for segmentation'
+                }
+            
+            for cat_col in categorical_cols:
+                try:
+                    if df[cat_col].isnull().sum() > 0.5 * len(df):
+                        continue
+                        
+                    df_temp = df.copy()
+                    df_temp[cat_col] = df_temp[cat_col].fillna('Missing')
+                    
+                    categories = df_temp[cat_col].value_counts().head(10).index.tolist()
+                    
+                    category_results = {}
+                    
+                    for category in categories:
+                        subset = df_temp[df_temp[cat_col] == category]
+                        
+                        if len(subset) < 5: 
+                            continue
+                            
+                        missing_percentages = subset.drop(columns=[cat_col]).isnull().mean() * 100
+                        
+                        missing_percentages = missing_percentages[missing_percentages > 0]
+                        
+                        if not missing_percentages.empty:
+                            category_results[category] = {
+                                'sample_size': len(subset),
+                                'missing_percentages': missing_percentages.to_dict()
+                            }
+                    
+                    if category_results:
+                        results[cat_col] = category_results
+            
+                except Exception as e:
+                    logger.error(f"Error in segment_based_analysis: {str(e)}", log_type='data_quality_assessment', console=verbose)
+                    return {'status': 'error', 'error': str(e)}
+        
+            return {
+                'status': 'completed',
+                'segments_analyzed': len(results),
+                'details': results
+            }
+        except Exception as e:
+            logger.error(f"Error in segment_based_analysis: {str(e)}", log_type='data_quality_assessment', console=verbose)
+            return {'status': 'error', 'error': str(e)}
+        
+    def generate_summary(self, results: Dict, verbose=False) -> str:
+        """Generate a human-readable summary of missing data analysis results"""
+        summary_lines = ["# Missing Data Analysis Summary"]
+
+        try:
+            if results["data_loaded"]:
+                summary_lines.append("\n## Dataset Overview")
+                summary_lines.append(f"- Successfully loaded dataset with shape: {results['data_shape'][0]} rows × {results['data_shape'][1]} columns")
+            else:
+                summary_lines.append("\n## Error Loading Dataset")
+                summary_lines.append("- Failed to load the dataset. Please check the file path and format.")
+                return "\n".join(summary_lines)
+        except Exception as e:
+            logger.error(f"Error in dataset loading section: {str(e)}", log_type='data_quality_assessment', console=verbose)
+
+        try:
+            if results.get("missing_rates") and "error" not in results["missing_rates"]:
+                missing_rates = results["missing_rates"]
+                summary_lines.append("\n## Missing Values Overview")
+                summary_lines.append(f"- Overall missing data: {missing_rates['overall']['overall_missing_percentage']:.2f}% "
+                                    f"({missing_rates['overall']['total_missing']} out of {missing_rates['overall']['total_cells']} cells)")
+                
+                if missing_rates['per_column']:
+                    summary_lines.append("\n### Top columns with missing values:")
+                    top_missing = sorted(missing_rates['per_column'], key=lambda x: x['Missing Percentage'], reverse=True)[:5]
+                    for col in top_missing:
+                        if col['Missing Percentage'] > 0:
+                            summary_lines.append(f"- {col['Column']}: {col['Missing Percentage']:.2f}% ({col['Missing Count']} values)")
+        except Exception as e:
+            logger.error(f"Error in missing values overview section: {str(e)}", log_type='data_quality_assessment', console=verbose)
+
+        try:
+            little_test = results.get("little_mcar_test", {})
+            if little_test.get("status") == "completed":
+                summary_lines.append("\n## Little's MCAR Test Results")
+                if little_test.get("is_mcar") is not None:
+                    if little_test["is_mcar"]:
+                        summary_lines.append("- **Data appears to be Missing Completely At Random (MCAR)**")
+                        if "proportion_significant" in little_test:
+                            summary_lines.append(f"- Proportion of significant tests: {little_test['proportion_significant']:.2f}")
+                        summary_lines.append(f"- Evidence: {little_test.get('evidence_strength', 'Evidence supports MCAR')}")
+                    else:
+                        summary_lines.append("- **Data does not appear to be Missing Completely At Random (MCAR)**")
+                        if "proportion_significant" in little_test:
+                            summary_lines.append(f"- Proportion of significant tests: {little_test['proportion_significant']:.2f}")
+                        summary_lines.append(f"- Evidence: {little_test.get('evidence_strength', 'Evidence against MCAR')}")
+                else:
+                    summary_lines.append("- Could not determine MCAR status definitively")
+            elif little_test.get("status") == "skipped":
+                summary_lines.append("\n## Little's MCAR Test Results")
+                summary_lines.append(f"- Test skipped: {little_test.get('reason', 'Unknown reason')}")
+        except Exception as e:
+            logger.error(f"Error in Little's MCAR test section: {str(e)}", log_type='data_quality_assessment', console=verbose)
+
+        try:
+            mar_results = results.get("mar_correlations", {})
+            if mar_results and "error" not in mar_results:
+                summary_lines.append("\n## Missing At Random (MAR) Analysis")
+                if mar_results.get("is_mar"):
+                    summary_lines.append("- **Evidence found that data is Missing At Random (MAR)**")
+                    summary_lines.append("- Missingness in some variables depends on observed values in other variables")
+                    mar_details = mar_results.get("details", {})
+                    for col in list(mar_details.keys())[:3]:
+                        related_cols = list(mar_details[col].keys())
+                        summary_lines.append(f"- Missingness in '{col}' depends on values in: {', '.join(related_cols[:3])}"
+                                            + (f" and {len(related_cols) - 3} more columns" if len(related_cols) > 3 else ""))
+                else:
+                    summary_lines.append("- No clear evidence that data is Missing At Random (MAR)")
+        except Exception as e:
+            logger.error(f"Error in MAR analysis section: {str(e)}", log_type='data_quality_assessment', console=verbose)
+
+        try:
+            mutual_results = results.get("mutual_missingness", {})
+            if mutual_results.get("status") == "completed":
+                summary_lines.append("\n## Mutual Missingness Analysis")
+                strong_corrs = mutual_results.get("strong_correlations", [])
+                if strong_corrs:
+                    summary_lines.append("- **Some variables tend to be missing together**")
+                    for corr in strong_corrs[:3]:
+                        summary_lines.append(f"- '{corr['column1']}' and '{corr['column2']}' have strongly correlated missingness (r = {corr['correlation']:.2f})")
+                    if len(strong_corrs) > 3:
+                        summary_lines.append(f"- {len(strong_corrs) - 3} other pairs of columns with strongly correlated missingness")
+                else:
+                    summary_lines.append("- No strong patterns of mutual missingness detected")
+
+                co_occurrence = mutual_results.get("co_occurrence", [])
+                if co_occurrence:
+                    summary_lines.append("\n### Most common co-occurrences of missing values:")
+                    for co in co_occurrence[:3]:
+                        summary_lines.append(f"- '{co['column1']}' and '{co['column2']}' are missing together in {co['both_missing_count']} rows")
+        except Exception as e:
+            logger.error(f"Error in mutual missingness section: {str(e)}", log_type='data_quality_assessment', console=verbose)
+
+        try:
+            segment_results = results.get("segment_analysis", {})
+            if segment_results.get("status") == "completed":
+                summary_lines.append("\n## Segment-based Missing Value Analysis")
+                segments = segment_results.get("details", {})
+                if segments:
+                    summary_lines.append(f"- Analyzed missingness patterns across {len(segments)} different segmentations")
+                    example_segment = list(segments.keys())[0]
+                    summary_lines.append(f"\n### Example: Missingness by '{example_segment}' categories")
+                    for category in list(segments[example_segment].keys())[:3]:
+                        cat_data = segments[example_segment][category]
+                        sample_size = cat_data['sample_size']
+                        missing_data = cat_data['missing_percentages']
+                        top_missing = sorted(missing_data.items(), key=lambda x: x[1], reverse=True)[:2]
+                        if top_missing:
+                            summary_lines.append(f"- In category '{category}' (n={sample_size}):")
+                            for col, pct in top_missing:
+                                summary_lines.append(f"  - '{col}' has {pct:.1f}% missing values")
+                else:
+                    summary_lines.append("- No significant segment-based missingness patterns found")
+        except Exception as e:
+            logger.error(f"Error in segment-based analysis section: {str(e)}", log_type='data_quality_assessment', console=verbose)
+
+        try:
+            summary_lines.append("\n## Recommendations for Missing Data")
+            if results.get("missing_rates") and "error" not in results["missing_rates"]:
+                missing_rates = results["missing_rates"]
+                high_missing_cols = [col for col in missing_rates['per_column'] if col['Missing Percentage'] > 50]
+                if high_missing_cols:
+                    summary_lines.append("- **Consider dropping columns** with high missing rates (>50%):")
+                    for col in high_missing_cols[:3]:
+                        summary_lines.append(f"  - '{col['Column']}' ({col['Missing Percentage']:.1f}% missing)")
+                    if len(high_missing_cols) > 3:
+                        summary_lines.append(f"  - and {len(high_missing_cols) - 3} other columns")
+
+            if results.get("little_mcar_test", {}).get("is_mcar") is True:
+                summary_lines.append("- Since data appears to be MCAR, **simple imputation** methods like mean/median/mode imputation are reasonable")
+
+            if results.get("mar_correlations", {}).get("is_mar") is True:
+                summary_lines.append("- Since evidence suggests data is MAR, consider **model-based imputation methods** like:")
+                summary_lines.append("  - Multiple Imputation by Chained Equations (MICE)")
+                summary_lines.append("  - K-Nearest Neighbors (KNN) imputation")
+                summary_lines.append("  - Regression-based imputation")
+
+            if results.get("mutual_missingness", {}).get("strong_correlations"):
+                summary_lines.append("- For variables that are missing together, consider **multivariate imputation** approaches")
+        except Exception as e:
+            logger.error(f"Error in recommendations section: {str(e)}", log_type='data_quality_assessment', console=verbose)
+
+        try:
+            if results.get("errors"):
+                summary_lines.append("\n## Analysis Issues")
+                summary_lines.append("The following issues were encountered during analysis:")
+                for error in results["errors"]:
+                    summary_lines.append(f"- {error}")
+        except Exception as e:
+            logger.error(f"Error in final error reporting section: {str(e)}", log_type='data_quality_assessment', console=verbose)
+
+        return "\n".join(summary_lines)
+
+    def agent_mva(self, report: str, verbose=False):
+        try:
+            kbc = KnowledgeBaseClass()
+            kb = kbc.initialize_knowledge_base(task_type=self.ml_task)
+            agent = kbc.initialize_agent(
+                agent_name="missing_value_analysis_agent",
+                llm_choice=self.llm_choice,
+                knowledge_base=kb
+            )
+
+            input = f"Task type: {self.ml_task}\n\n\n{report}"
+            res: RunResponse = agent.run(input, stream=False)
+
+            formatted_code = self.post_process_code(code=res.content.code_generated, verbose=verbose)
+
+            with open("temp.py", "w") as f:
+                f.write(f'''{formatted_code}''')
+
+            from temp import main
+
+            results = main(self.data_source)
+
+            return results
+        except Exception as e:
+            logger.error(f"Failed to run agentic analysis with error: {e}", log_type="data_quality_assessment", console=verbose)
+            return ""
+
+    def post_process_code(self, code: str, verbose=False) -> str:
+        def fix_fstring_quotes(match):
+            inner = match.group(1)
+            fixed_inner = re.sub(r'\{([^{}]*?)\["([^"]+)"\]\}', r"{\1['\2']}", inner)
+            return f'f"{fixed_inner}"'
+
+        fstring_pattern = r"f'([^']*{[^}]+}[^']*)'"
+        code = re.sub(fstring_pattern, fix_fstring_quotes, code)
+
+        try:
+            ast.parse(code)
+        except SyntaxError as e:
+            logger.error(f"[Syntax Error after fix] Line {e.lineno}: {e.msg}", log_type="data_quality_assessment", console=verbose)
+            pass
+
+        return code
+
+    def detect_duplicates(
+        self,
+        key_columns: Optional[List[str]] = None,
+        similarity_columns: Optional[List[str]] = None,
+        similarity_threshold: float = 0.8,
+        numeric_threshold: float = 0.05,
+        sample_size: Optional[int] = None,
+        verbose = False
+    ) -> Dict[str, Any]:
+        """Comprehensive duplicate detection function that can analyze any CSV dataset"""
+        
+        df = self.data
+
+        results = {
+            "dataset_info": {
+                "original_rows": len(df),
+                "original_columns": len(df.columns),
+                "column_dtypes": {col: str(df[col].dtype) for col in df.columns}
+            },
+            "exact_duplicates": {},
+            "key_based_duplicates": {},
+            "near_duplicates": {},
+            "distribution_impact": {},
+            "summary": ""
+        }
+        
+        if sample_size and len(df) > sample_size:
+            try:
+                analysis_df = df.sample(sample_size, random_state=42)
+                results["dataset_info"]["sampled"] = True
+                results["dataset_info"]["sample_size"] = sample_size
+            except Exception as e:
+                analysis_df = df
+                results["dataset_info"]["sampling_error"] = str(e)
+        else:
+            analysis_df = df
+        
+        try:
+            exact_duplicates = self.analyze_exact_duplicates(analysis_df, verbose=verbose)
+            results["exact_duplicates"] = exact_duplicates
+        except Exception as e:
+            results["exact_duplicates"]["error"] = str(e)
+        
+        try:
+            key_duplicates = self.analyze_key_based_duplicates(analysis_df, key_columns, verbose=verbose)
+            results["key_based_duplicates"] = key_duplicates
+        except Exception as e:
+            results["key_based_duplicates"]["error"] = str(e)
+        
+        try:
+            near_duplicates = self.analyze_near_duplicates(analysis_df, similarity_columns, 
+                                                    similarity_threshold, numeric_threshold, verbose=verbose)
+            results["near_duplicates"] = near_duplicates
+        except Exception as e:
+            results["near_duplicates"]["error"] = str(e)
+        
+        try:
+            distribution_impact = self.analyze_distribution_impact(analysis_df, results, verbose=verbose)
+            results["distribution_impact"] = distribution_impact
+        except Exception as e:
+            results["distribution_impact"]["error"] = str(e)
+        
+        try:
+            results["summary"] = self.generate_duplicate_analysis_summary(results, verbose=verbose)
+        except Exception as e:
+            results["summary"] = f"Error generating summary: {str(e)}"
+        
+        return results
+
+    def analyze_exact_duplicates(self, df: pd.DataFrame, verbose=False) -> Dict[str, Any]:
+        """Detect and analyze exact duplicate rows"""
+        results = {}
+        
+        try:
+            duplicate_mask = df.duplicated()
+            duplicated_rows = df[duplicate_mask]
+            
+            unique_duplicate_patterns = df[df.duplicated(keep=False)].drop_duplicates()
+            
+            results["total_exact_duplicates"] = int(duplicate_mask.sum())
+            results["unique_duplicate_patterns"] = len(unique_duplicate_patterns)
+            results["duplicate_percentage"] = round(results["total_exact_duplicates"] / len(df) * 100, 2)
+            
+            if len(duplicated_rows) > 0:
+                dup_counts = Counter(map(tuple, df[df.duplicated(keep=False)].itertuples(index=False)))
+                most_common = [(str(k), v) for k, v in dup_counts.most_common(5)]
+                results["most_common_duplicates"] = most_common
+
+            if len(df.columns) <= 20:
+                column_duplication = {}
+                for col in df.columns:
+                    dup_count = df.duplicated(subset=[col], keep=False).sum()
+                    if dup_count > 0:
+                        column_duplication[col] = int(dup_count)
+                
+                results["column_duplication_counts"] = column_duplication
+        
+        except Exception as e:
+            results["error"] = f"Error in exact duplicates analysis: {str(e)}"
+            logger.error(f"Error in exact duplicates analysis: {str(e)}", log_type="data_quality_assessment", console=verbose)
+        
+        return results
+
+    def analyze_key_based_duplicates(self, df: pd.DataFrame, key_columns: Optional[List[str]] = None, verbose=False) -> Dict[str, Any]:
+        """Detect and analyze key-based duplicates"""
+        results = {}
+        
+        try:
+            if key_columns is None:
+                key_columns = self.identify_key_candidates(df)
+                results["detected_key_candidates"] = key_columns
+            
+            if not key_columns:
+                results["message"] = "No key columns identified or provided"
+                return results
+                
+            dup_counts = {}
+            for key in key_columns:
+                try:
+                    if key in df.columns:
+                        dups = df.duplicated(subset=[key], keep=False)
+                        dup_count = int(dups.sum())
+                        dup_percentage = round(dup_count / len(df) * 100, 2)
+                        
+                        if dup_count > 0:
+                            dup_values = df[dups][key].value_counts().head(5).to_dict()
+                            dup_values = {str(k): int(v) for k, v in dup_values.items()}
+                        else:
+                            dup_values = {}
+                        
+                        dup_counts[key] = {
+                            "duplicate_count": dup_count,
+                            "duplicate_percentage": dup_percentage,
+                            "top_duplicated_values": dup_values
+                        }
+                except Exception as e:
+                    dup_counts[key] = {"error": str(e)}
+            
+            results["key_duplicate_analysis"] = dup_counts
+            
+            if len(key_columns) > 1:
+                try:
+                    multi_key_dups = df.duplicated(subset=key_columns, keep=False)
+                    results["multi_key_duplicates"] = {
+                        "count": int(multi_key_dups.sum()),
+                        "percentage": round(multi_key_dups.sum() / len(df) * 100, 2)
+                    }
+                except Exception as e:
+                    results["multi_key_duplicates"] = {"error": str(e)}
+        
+        except Exception as e:
+            results["error"] = f"Error in key-based duplicates analysis: {str(e)}"
+            logger.error(f"Error in key-based duplicates analysis: {str(e)}", log_type="data_quality_assessment", console=verbose)
+        
+        return results
+
+    def identify_key_candidates(self, df: pd.DataFrame) -> List[str]:
+        """
+        Identify potential primary key columns in the dataframe
+        """
+        candidates = []
+        
+        try:
+            for col in df.columns:
+                if df[col].nunique() == len(df) and df[col].notna().all():
+                    candidates.append(col)
+            
+            if not candidates:
+                for col in df.columns:
+                    uniqueness_ratio = df[col].nunique() / len(df)
+                    if uniqueness_ratio > 0.9 and df[col].notna().all():
+                        candidates.append(col)
+            
+            if not candidates:
+                for col in df.columns:
+                    if any(key_term in col.lower() for key_term in ['id', 'key', 'code', 'num', 'uuid']):
+                        candidates.append(col)
+        
+        except Exception:
+            pass
+        
+        return candidates
+
+    def analyze_near_duplicates(
+        self,
+        df: pd.DataFrame, 
+        similarity_columns: Optional[List[str]] = None,
+        similarity_threshold: float = 0.8,
+        numeric_threshold: float = 0.05,
+        verbose = False
+    ) -> Dict[str, Any]:
+        """
+        Detect and analyze near-duplicate rows using similarity metrics
+        """
+        results = {}
+        
+        try:
+            if similarity_columns is None:
+                text_columns = [col for col in df.columns if df[col].dtype == 'object']
+                numeric_columns = [col for col in df.columns if pd.api.types.is_numeric_dtype(df[col])]
+                
+                similarity_columns = text_columns[:5] if len(text_columns) > 5 else text_columns
+                results["auto_selected_text_columns"] = similarity_columns
+                results["available_numeric_columns"] = numeric_columns
+            
+            if not similarity_columns:
+                results["message"] = "No text columns identified for similarity analysis"
+                return results
+            
+            sample_size = min(1000, len(df))
+            if len(df) > sample_size:
+                sample_df = df.sample(sample_size, random_state=42)
+                results["sampled_for_similarity"] = True
+                results["similarity_sample_size"] = sample_size
+            else:
+                sample_df = df
+                
+            text_similarity_results = {}
+            for col in similarity_columns:
+                try:
+                    if col in df.columns and df[col].dtype == 'object':
+                        col_data = sample_df[col].fillna("").astype(str)
+                        
+                        if col_data.nunique() <= 1:
+                            continue
+                        
+                        similar_pairs = self.find_similar_text(col_data, similarity_threshold)
+                        
+                        if similar_pairs:
+                            text_similarity_results[col] = {
+                                "similar_pairs_count": len(similar_pairs),
+                                "examples": similar_pairs[:5]  # Limit to first 5 examples
+                            }
+                except Exception as e:
+                    text_similarity_results[col] = {"error": str(e)}
+            
+            results["text_similarity"] = text_similarity_results
+            
+            numeric_similarity_results = {}
+            numeric_cols = [col for col in df.columns if col in similarity_columns and pd.api.types.is_numeric_dtype(df[col])]
+            
+            for col in numeric_cols:
+                try:
+                    if df[col].isna().sum() / len(df) > 0.3:  # More than 30% missing
+                        continue
+                        
+                    col_data = sample_df[col].dropna()
+                    similar_numeric_pairs = self.find_similar_numeric(col_data, numeric_threshold)
+                    
+                    if similar_numeric_pairs:
+                        numeric_similarity_results[col] = {
+                            "similar_pairs_count": len(similar_numeric_pairs),
+                            "examples": similar_numeric_pairs[:5]  # Limit to first 5 examples
+                        }
+                except Exception as e:
+                    numeric_similarity_results[col] = {"error": str(e)}
+                    
+            results["numeric_similarity"] = numeric_similarity_results
+            
+            try:
+                if len(similarity_columns) >= 2:
+                    combined_text = sample_df[similarity_columns].fillna("").astype(str).apply(
+                        lambda x: " ".join(x), axis=1
+                    )
+                    
+                    if len(combined_text) > 1:  # Need at least 2 rows for comparison
+                        vectorizer = TfidfVectorizer(min_df=1, stop_words='english')
+                        try:
+                            tfidf_matrix = vectorizer.fit_transform(combined_text)
+                            similar_doc_pairs = self.find_similar_vectors(tfidf_matrix, similarity_threshold)
+                            
+                            if similar_doc_pairs:
+                                results["multi_column_similarity"] = {
+                                    "similar_rows_count": len(similar_doc_pairs),
+                                    "examples": similar_doc_pairs[:5]  # Limit to first 5 examples
+                                }
+                        except Exception as e:
+                            results["multi_column_similarity"] = {"error": str(e)}
+            except Exception as e:
+                results["multi_column_similarity_error"] = str(e)
+        
+        except Exception as e:
+            results["error"] = f"Error in near-duplicates analysis: {str(e)}"
+            logger.error(f"Error in near-duplicates analysis: {str(e)}", log_type="data_quality_assessment", console=verbose)
+
+        
+        return results
+
+    def find_similar_text(self, series: pd.Series, threshold: float) -> List[Tuple[str, str, float]]:
+        """
+        Find similar text pairs in a series
+        """
+        similar_pairs = []
+        values = series.tolist()
+        
+        max_comparisons = 10000
+        if len(values) > 200:  # For large sets, sample comparisons
+            import random
+            from itertools import combinations
+            
+            indices = list(range(len(values)))
+            all_pairs = list(combinations(indices, 2))
+            
+            if len(all_pairs) > max_comparisons:
+                sampled_pairs = random.sample(all_pairs, max_comparisons)
+            else:
+                sampled_pairs = all_pairs
+                
+            for i, j in sampled_pairs:
+                try:
+                    if not values[i] or not values[j]:
+                        continue
+                        
+                    similarity = difflib.SequenceMatcher(None, values[i], values[j]).ratio()
+                    if similarity >= threshold:
+                        similar_pairs.append((values[i], values[j], round(similarity, 2)))
+                except:
+                    continue
+        else:
+            for i in range(len(values)):
+                try:
+                    for j in range(i+1, len(values)):
+                        if not values[i] or not values[j]:
+                            continue
+                            
+                        similarity = difflib.SequenceMatcher(None, values[i], values[j]).ratio()
+                        if similarity >= threshold:
+                            similar_pairs.append((values[i], values[j], round(similarity, 2)))
+                except:
+                    continue
+        return similar_pairs
+
+    def find_similar_numeric(self, series: pd.Series, threshold: float) -> List[Tuple[float, float, float]]:
+        """
+        Find similar numeric pairs in a series
+        """
+        similar_pairs = []
+        values = series.tolist()
+        
+        max_comparisons = 10000
+        if len(values) > 200:  # For large sets, sample comparisons
+            import random
+            from itertools import combinations
+            
+            indices = list(range(len(values)))
+            all_pairs = list(combinations(indices, 2))
+            
+            if len(all_pairs) > max_comparisons:
+                sampled_pairs = random.sample(all_pairs, max_comparisons)
+            else:
+                sampled_pairs = all_pairs
+                
+            for i, j in sampled_pairs:
+                try:
+                    if values[i] == 0 or values[j] == 0:
+                        continue
+                        
+                    max_val = max(abs(values[i]), abs(values[j]))
+                    min_val = min(abs(values[i]), abs(values[j]))
+                    
+                    if max_val == 0:  # Both values are zero
+                        continue
+                        
+                    rel_diff = (max_val - min_val) / max_val
+                    
+                    if rel_diff <= threshold:
+                        similar_pairs.append((values[i], values[j], round(rel_diff, 3)))
+                except:
+                    continue
+        else:
+            for i in range(len(values)):
+                try: 
+                    for j in range(i+1, len(values)):
+                        if values[i] == 0 or values[j] == 0:
+                            continue
+                            
+                        max_val = max(abs(values[i]), abs(values[j]))
+                        min_val = min(abs(values[i]), abs(values[j]))
+                        
+                        if max_val == 0:  # Both values are zero
+                            continue
+                            
+                        rel_diff = (max_val - min_val) / max_val
+                        
+                        if rel_diff <= threshold:
+                            similar_pairs.append((values[i], values[j], round(rel_diff, 3)))
+                except:
+                    continue
+        
+        return similar_pairs
+
+    def find_similar_vectors(self, tfidf_matrix, threshold: float) -> List[Tuple[int, int, float]]:
+        """
+        Find similar document pairs based on TF-IDF vectors
+        """
+        similar_pairs = []
+        
+        max_rows = 1000
+        if tfidf_matrix.shape[0] > max_rows:
+            import random
+            indices = random.sample(range(tfidf_matrix.shape[0]), max_rows)
+            sampled_matrix = tfidf_matrix[indices]
+        else:
+            sampled_matrix = tfidf_matrix
+            indices = list(range(tfidf_matrix.shape[0]))
+        
+        similarity_matrix = cosine_similarity(sampled_matrix)
+        
+        rows, cols = np.where(similarity_matrix >= threshold)
+        for i, j in zip(rows, cols):
+            try:
+                if i < j:  # Only include each pair once
+                    similar_pairs.append((indices[i], indices[j], round(similarity_matrix[i, j], 2)))
+            except:
+                continue
+        
+        return similar_pairs
+
+    def analyze_distribution_impact(self, df: pd.DataFrame, results: Dict[str, Any], verbose=False) -> Dict[str, Any]:
+        """
+        Assess impact of duplicates on distribution statistics
+        """
+        distribution_results = {}
+        
+        try:
+            df_deduped = df.drop_duplicates()
+            
+            if len(df) == len(df_deduped):
+                distribution_results["message"] = "No exact duplicates found for distribution impact analysis"
+                return distribution_results
+            
+            numeric_cols = [col for col in df.columns if pd.api.types.is_numeric_dtype(df[col])]
+            
+            col_impacts = {}
+            for col in numeric_cols:
+                try:
+                    if df[col].isna().sum() / len(df) > 0.5:  # More than 50% missing
+                        continue
+                        
+                    orig_stats = {
+                        "mean": float(df[col].mean()),
+                        "median": float(df[col].median()),
+                        "std": float(df[col].std()),
+                        "min": float(df[col].min()),
+                        "max": float(df[col].max())
+                    }
+                    
+                    dedup_stats = {
+                        "mean": float(df_deduped[col].mean()),
+                        "median": float(df_deduped[col].median()),
+                        "std": float(df_deduped[col].std()),
+                        "min": float(df_deduped[col].min()),
+                        "max": float(df_deduped[col].max())
+                    }
+                    
+                    pct_changes = {}
+                    for stat in orig_stats:
+                        try:
+                            if orig_stats[stat] != 0:
+                                pct_changes[stat] = round(
+                                    (dedup_stats[stat] - orig_stats[stat]) / abs(orig_stats[stat]) * 100, 2
+                                )
+                            else:
+                                pct_changes[stat] = 0.0 if dedup_stats[stat] == 0 else float('inf')
+                        except:
+                            continue
+                    
+                    col_impacts[col] = {
+                        "original_stats": orig_stats,
+                        "deduped_stats": dedup_stats,
+                        "percentage_changes": pct_changes,
+                        "significant_change": any(abs(chg) > 5 for chg in pct_changes.values())
+                    }
+                except Exception as e:
+                    col_impacts[col] = {"error": str(e)}
+            
+            distribution_results["numeric_columns_impact"] = col_impacts
+            
+            cat_cols = [col for col in df.columns if df[col].dtype == 'object']
+            
+            cat_cols = cat_cols[:10] if len(cat_cols) > 10 else cat_cols
+            
+            cat_impacts = {}
+            for col in cat_cols:
+                try:
+                    orig_counts = df[col].value_counts(normalize=True).head(10).to_dict()
+                    dedup_counts = df_deduped[col].value_counts(normalize=True).head(10).to_dict()
+                    
+                    orig_counts = {str(k): float(v) for k, v in orig_counts.items()}
+                    dedup_counts = {str(k): float(v) for k, v in dedup_counts.items()}
+                    
+                    changed_cats = {}
+                    all_cats = set(list(orig_counts.keys()) + list(dedup_counts.keys()))
+                    
+                    for cat in all_cats:
+                        orig_val = orig_counts.get(cat, 0)
+                        dedup_val = dedup_counts.get(cat, 0)
+                        
+                        if orig_val > 0:
+                            pct_change = round((dedup_val - orig_val) / orig_val * 100, 2)
+                            if abs(pct_change) > 5:  # 5% threshold for significant change
+                                changed_cats[cat] = pct_change
+                    
+                    cat_impacts[col] = {
+                        "significant_category_changes": changed_cats,
+                        "has_significant_changes": len(changed_cats) > 0
+                    }
+                except Exception as e:
+                    cat_impacts[col] = {"error": str(e)}
+            
+            distribution_results["categorical_columns_impact"] = cat_impacts
+            
+            try:
+                significant_numeric_changes = sum(
+                    1 for col in col_impacts if "significant_change" in col_impacts[col] and col_impacts[col]["significant_change"]
+                )
+                
+                significant_cat_changes = sum(
+                    1 for col in cat_impacts if "has_significant_changes" in cat_impacts[col] and cat_impacts[col]["has_significant_changes"]
+                )
+                
+                distribution_results["overall_assessment"] = {
+                    "columns_with_significant_numeric_changes": significant_numeric_changes,
+                    "columns_with_significant_categorical_changes": significant_cat_changes,
+                    "total_columns_analyzed": len(col_impacts) + len(cat_impacts),
+                    "duplicates_impact_level": self.get_impact_level(
+                        significant_numeric_changes + significant_cat_changes, 
+                        len(col_impacts) + len(cat_impacts)
+                    )
+                }
+            except Exception as e:
+                distribution_results["overall_assessment"] = {"error": str(e)}
+        
+        except Exception as e:
+            distribution_results["error"] = f"Error in distribution impact analysis: {str(e)}"
+            logger.error(f"Error in distribution impact analysis: {str(e)}", log_type="data_quality_assessment", console=verbose)
+        
+        return distribution_results
+
+    def get_impact_level(self, significant_changes: int, total_cols: int) -> str:
+        """
+        Determine the impact level based on the proportion of columns with significant changes
+        """
+        if total_cols == 0:
+            return "Unknown"
+        try:
+            proportion = significant_changes / total_cols
+            
+            if proportion == 0:
+                return "None"
+            elif proportion < 0.1:
+                return "Minimal"
+            elif proportion < 0.3:
+                return "Low"
+            elif proportion < 0.5:
+                return "Moderate"
+            elif proportion < 0.7:
+                return "High"
+            else:
+                return "Severe"
+        except:
+            pass
+
+    def generate_duplicate_analysis_summary(self, results: Dict[str, Any], verbose=False) -> str:
+            """
+            Generate a text summary of duplicate detection results
+            """
+            summary_parts = []
+
+            try:
+                dataset_info = results.get("dataset_info", {})
+                summary_parts.append(f"Dataset Summary: {dataset_info.get('original_rows', 'N/A')} rows, "
+                                    f"{dataset_info.get('original_columns', 'N/A')} columns")
+                if dataset_info.get("sampled"):
+                    summary_parts.append(f"Analysis performed on a sample of {dataset_info.get('sample_size', 'N/A')} rows")
+            except Exception as e:
+                summary_parts.append(f"Error summarizing dataset info: {e}")
+
+            try:
+                exact_dups = results.get("exact_duplicates", {})
+                if "error" in exact_dups:
+                    summary_parts.append(f"Error in exact duplicate analysis summary: {exact_dups['error']}")
+                else:
+                    dup_count = exact_dups.get("total_exact_duplicates", 0)
+                    dup_pct = exact_dups.get("duplicate_percentage", 0)
+                    summary_parts.append(f"Exact Duplicates: {dup_count} rows ({dup_pct}% of dataset)")
+            except Exception as e:
+                summary_parts.append(f"Error summarizing exact duplicates: {e}")
+
+            try:
+                key_dups = results.get("key_based_duplicates", {})
+                if "error" in key_dups:
+                    summary_parts.append(f"Error in key-based duplicate analysis summary: {key_dups['error']}")
+                else:
+                    if "detected_key_candidates" in key_dups:
+                        candidates = key_dups.get("detected_key_candidates", [])
+                        if candidates:
+                            summary_parts.append(f"Detected Key Candidates: {', '.join(candidates)}")
+                        else:
+                            summary_parts.append("No key candidates detected")
+                    key_analysis = key_dups.get("key_duplicate_analysis", {})
+                    if key_analysis:
+                        key_summary = []
+                        for key, info in key_analysis.items():
+                            if "error" in info:
+                                continue
+                            key_summary.append(f"{key}: {info.get('duplicate_count', 0)} duplicates "
+                                            f"({info.get('duplicate_percentage', 0)}%)")
+                        if key_summary:
+                            summary_parts.append("Key-Based Duplicates Summary:\n- " + "\n- ".join(key_summary))
+                    multi_key_dups = key_dups.get("multi_key_duplicates", {})
+                    if multi_key_dups and "error" not in multi_key_dups:
+                        summary_parts.append(f"Multi-Column Key Duplicates: {multi_key_dups.get('count', 0)} rows "
+                                            f"({multi_key_dups.get('percentage', 0)}%)")
+            except Exception as e:
+                summary_parts.append(f"Error summarizing key-based duplicates: {e}")
+
+            try:
+                near_dups = results.get("near_duplicates", {})
+                if "error" in near_dups:
+                    summary_parts.append(f"Error in near-duplicate analysis summary: {near_dups['error']}")
+                else:
+                    text_similarity = near_dups.get("text_similarity", {})
+                    if text_similarity:
+                        text_similar_count = sum(info.get("similar_pairs_count", 0)
+                                                for info in text_similarity.values()
+                                                if "error" not in info)
+                        if text_similar_count > 0:
+                            summary_parts.append(f"Text Near-Duplicates: {text_similar_count} similar pairs identified")
+
+                    numeric_similarity = near_dups.get("numeric_similarity", {})
+                    if numeric_similarity:
+                        numeric_similar_count = sum(info.get("similar_pairs_count", 0)
+                                                    for info in numeric_similarity.values()
+                                                    if "error" not in info)
+                        if numeric_similar_count > 0:
+                            summary_parts.append(f"Numeric Near-Duplicates: {numeric_similar_count} similar pairs identified")
+
+                    multi_col = near_dups.get("multi_column_similarity", {})
+                    if multi_col and "error" not in multi_col and multi_col.get("similar_rows_count", 0) > 0:
+                        summary_parts.append(f"Multi-Column Near-Duplicates: {multi_col.get('similar_rows_count', 0)} similar row pairs")
+            except Exception as e:
+                summary_parts.append(f"Error summarizing near duplicates: {e}")
+
+            try:
+                dist_impact = results.get("distribution_impact", {})
+                if "error" in dist_impact:
+                    summary_parts.append(f"Error in distribution impact analysis summary: {dist_impact['error']}")
+                else:
+                    overall = dist_impact.get("overall_assessment", {})
+                    if overall and "error" not in overall:
+                        impact_level = overall.get("duplicates_impact_level", "Unknown")
+                        sig_cols = overall.get("columns_with_significant_numeric_changes", 0) + \
+                                    overall.get("columns_with_significant_categorical_changes", 0)
+                        total_cols = overall.get("total_columns_analyzed", 0)
+                        if total_cols > 0:
+                            summary_parts.append(f"Distribution Impact: {impact_level} "
+                                                f"({sig_cols}/{total_cols} columns significantly affected)")
+            except Exception as e:
+                summary_parts.append(f"Error summarizing distribution impact: {e}")
+
+            try:
+                if "exact_duplicates" in results and "total_exact_duplicates" in results["exact_duplicates"]:
+                    dup_pct = results["exact_duplicates"].get("duplicate_percentage", 0)
+
+                    if dup_pct > 20:
+                        summary_parts.append("\nRECOMMENDATION: High duplicate percentage detected. "
+                                            "Consider deduplicating the dataset before analysis.")
+                    elif dup_pct > 5:
+                        summary_parts.append("\nRECOMMENDATION: Moderate duplicate percentage detected. "
+                                            "Consider the impact of duplicates on your analysis.")
+                    elif dup_pct > 0:
+                        summary_parts.append("\nRECOMMENDATION: Low duplicate percentage detected. "
+                                            "Minimal impact expected on analysis.")
+                    else:
+                        summary_parts.append("\nRECOMMENDATION: No exact duplicates found.")
+                else:
+                    summary_parts.append("\nRECOMMENDATION: Duplicate analysis complete.")
+            except Exception as e:
+                summary_parts.append(f"Error generating recommendation: {e}")
+
+            return "\n".join(summary_parts)
+
+    def perform_consistency_checks(self, verbose=False) -> Dict[str, Any]:
+            """Performs various data consistency checks."""
+            results = {}
+
+            try:
+                results["cross_field_validation"] = self.check_cross_field_validity()
+            except Exception as e:
+                results["cross_field_validation"] = {"error": str(e)}
+                logger.error(f"Error in cross-field validation: {e}", log_type="consistency_check", console=verbose)
+
+            try:
+                results["logical_relationship"] = self.verify_logical_relationships()
+            except Exception as e:
+                results["logical_relationship"] = {"error": str(e)}
+                logger.error(f"Error in logical relationship verification: {e}", log_type="consistency_check", console=verbose)
+
+            try:
+                results["data_type_validation"] = self.validate_data_types()
+            except Exception as e:
+                results["data_type_validation"] = {"error": str(e)}
+                logger.error(f"Error in data type validation: {e}", log_type="consistency_check", console=verbose)
+
+            try:
+                results["value_transition_validity"] = self.check_value_transitions()
+            except Exception as e:
+                results["value_transition_validity"] = {"error": str(e)}
+                logger.error(f"Error in value transition validity: {e}", log_type="consistency_check", console=verbose)
+
+            try:
+                results["unit_consistency"] = self.check_unit_consistency()
+            except Exception as e:
+                results["unit_consistency"] = {"error": str(e)}
+                logger.error(f"Error in unit consistency checks: {e}", log_type="consistency_check", console=verbose)
+
+            try:
+                results["format_consistency"] = self.check_format_consistency()
+            except Exception as e:
+                results["format_consistency"] = {"error": str(e)}
+                logger.error(f"Error in format consistency tests: {e}", log_type="consistency_check", console=verbose)
+
+            return results
+
+    def check_cross_field_validity(self) -> Dict[str, Any]:
+        """Example: Check if discount is not greater than the price."""
+        if self.data is None or 'price' not in self.data.columns or 'discount' not in self.data.columns:
+            return {"warning": "Price or discount columns not found."}
+        invalid_rows = self.data[self.data['discount'] > self.data['price']]
+        return {"invalid_count": len(invalid_rows), "invalid_indices": invalid_rows.index.tolist()}
+
+    def verify_logical_relationships(self) -> Dict[str, Any]:
+        """Example: If order_status is 'shipped', shipment_date should not be NaN."""
+        if self.data is None or 'order_status' not in self.data.columns or 'shipment_date' not in self.data.columns:
+            return {"warning": "order_status or shipment_date columns not found."}
+        invalid_rows = self.data[(self.data['order_status'] == 'shipped') & (self.data['shipment_date'].isnull())]
+        return {"invalid_count": len(invalid_rows), "invalid_indices": invalid_rows.index.tolist()}
+
+    def validate_data_types(self) -> Dict[str, Any]:
+        """Check if declared data types match actual data types (basic check)."""
+        if self.data is None:
+            return {"warning": "No data loaded."}
+        mismatches = {}
+        for col in self.data.columns:
+            try:
+                inferred_type = pd.api.types.infer_dtype(self.data[col])
+                mismatches[col] = {"inferred_type": inferred_type}
+            except:
+                continue
+        return mismatches
+
+    def check_value_transitions(self) -> Dict[str, Any]:
+        """Example: Check if a 'temperature' column generally increases over a 'time' column (simplistic)."""
+        if self.data is None or 'temperature' not in self.data.columns or 'time' not in self.data.columns:
+            return {"warning": "temperature or time columns not found."}
+        diffs = self.data['temperature'].diff()
+        decreasing_transitions = diffs[diffs < 0].count()
+        increasing_transitions = diffs[diffs > 0].count()
+        return {"decreasing_transitions": decreasing_transitions, "increasing_transitions": increasing_transitions}
+
+    def check_unit_consistency(self) -> Dict[str, Any]:
+        """Example: Check if a 'measurement' column has consistent units (requires some form of unit identification)."""
+        if self.data is None or 'measurement' not in self.data.columns:
+            return {"warning": "measurement column not found."}
+        
+        unique_values = self.data['measurement'].astype(str).unique()
+        return {"unique_measurement_values": list(unique_values), "comment": "Implement more sophisticated unit parsing if needed."}
+
+    def check_format_consistency(self) -> Dict[str, Any]:
+        """Example: Check if a 'date' column has a consistent date format."""
+        if self.data is None or 'date' not in self.data.columns:
+            return {"warning": "date column not found."}
+        formats = set()
+        for value in self.data['date'].astype(str).dropna().unique():
+            try:
+                pd.to_datetime(value)
+            except:
+                formats.add(f"Inconsistent format: '{value}'")
+        return {"inconsistent_formats": list(formats)}
+
+    def perform_data_completeness_checks(self, verbose=False) -> Dict[str, Any]:
+        """Performs various data completeness checks."""
+        results = {}
+
+        try:
+            results["coverage_analysis"] = self.analyze_coverage()
+        except Exception as e:
+            results["coverage_analysis"] = {"error": str(e)}
+            logger.error(f"Error in coverage analysis: {e}", log_type="completeness_check", console=verbose)
+
+        try:
+            results["time_period_completeness"] = self.check_time_completeness()
+        except Exception as e:
+            results["time_period_completeness"] = {"error": str(e)}
+            logger.error(f"Error in time period completeness checks: {e}", log_type="completeness_check", console=verbose)
+
+        return results
+
+    def analyze_coverage(self) -> Dict[str, Any]:
+        """Analyze the percentage of non-missing values for each column."""
+        if self.data is None:
+            return {"warning": "No data loaded."}
+        coverage = {}
+        for col in self.data.columns:
+            try:
+                coverage[col] = f"{self.data[col].count() / len(self.data) * 100:.2f}%"
+            except:
+                continue
+        return coverage
+
+    def check_time_completeness(self) -> Dict[str, Any]:
+        """Example: Check if there are missing dates in a 'timestamp' column (assuming sorted)."""
+        if self.data is None or 'timestamp' not in self.data.columns:
+            return {"warning": "timestamp column not found."}
+        try:
+            time_series = pd.to_datetime(self.data['timestamp']).sort_values()
+            if not time_series.empty:
+                first_date = time_series.iloc[0].date()
+                last_date = time_series.iloc[-1].date()
+                expected_days = (last_date - first_date).days + 1
+                actual_days = len(time_series.dt.date.unique())
+                return {"expected_unique_days": expected_days, "actual_unique_days": actual_days}
+            else:
+                return {"warning": "Timestamp column is empty."}
+        except Exception as e:
+            return {"error": f"Could not process timestamp column: {e}"}
+
+    def generate_consistency_analysis_summary(self, consistency_results: Dict[str, Any], completeness_results: Dict[str, Any]) -> str:
+        """Generates a summary of the data quality assessment."""
+        summary_lines = ["Data Quality Assessment Summary:\n"]
+
+        summary_lines.append("\nConsistency Checks:\n")
+        if consistency_results:
+            for check, result in consistency_results.items():
+                try:
+                    summary_lines.append(f"- {check}: {result}\n")
+                except:
+                    continue
+        else:
+            summary_lines.append("- No consistency checks performed or results available.\n")
+
+        summary_lines.append("\nData Completeness Checks:\n")
+        if completeness_results:
+            for check, result in completeness_results.items():
+                try:
+                    summary_lines.append(f"- {check}: {result}\n")
+                except:
+                    continue
+        else:
+            summary_lines.append("- No data completeness checks performed or results available.\n")
+
+        return "".join(summary_lines)
+
+    def generate_report_from_agent(self, input)->str:
+        '''Transform the json output to a user-readable report'''
+        try:
+            input = f"ML Task: {self.ml_task}\n{input}"
+            response: RunResponse = self.writer.run(input, stream=False)
+            return response.content
+        except Exception as e:
+            return f"Failed to generate report with error: {e}"
+
+    def convert_numpy_types(self, obj):
+        if isinstance(obj, dict):
+            return {k: self.convert_numpy_types(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [self.convert_numpy_types(item) for item in obj]
+        elif isinstance(obj, np.integer):
+            return int(obj)
+        elif isinstance(obj, np.floating):
+            return float(obj)
+        elif isinstance(obj, np.bool_):
+            return bool(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        else:
+            return obj
+
+
+    def run(self, verbose=False) -> Dict[str, dict]:
+        '''Run the entire workflow'''
+        logger.info("Starting missing value analysis...", log_type="data_quality_assessment", console=verbose)
+        mva_results = self.analyze_missing_data(verbose=verbose)
+        # logger.info("Starting agentic missing value analysis...", log_type="data_quality_assessment", console=verbose)
+        # agentic_mva = self.agent_mva(report=mva_results['summary'], verbose=verbose)
+        
+        logger.info("Starting Duplicate value analysis...", log_type="data_quality_assessment", console=verbose)
+        duplicate_analysis_results = self.detect_duplicates(verbose=verbose)
+
+        logger.info("Starting Data quality & Consistency checks...", log_type="data_quality_assessment", console=verbose)
+        consistency_results = self.perform_consistency_checks(verbose=verbose)
+        completeness_results = self.perform_data_completeness_checks(verbose=verbose)
+        data_quality_results = consistency_results | completeness_results
+        data_quality_summary = self.generate_consistency_analysis_summary(consistency_results, completeness_results)
+
+        logger.info("Generating final reports....", log_type='data_quality_assessment', console=verbose)
+        
+        final_result = {
+            "missing_value_analysis": {
+                'dict': self.convert_numpy_types(mva_results),
+                'report': ""
+            },
+            "duplicate_analysis": {
+                'dict': self.convert_numpy_types(duplicate_analysis_results),
+                'report': ""
+            },
+            "data_quality_analysis": {
+                'dict': self.convert_numpy_types(data_quality_results),
+                'report': ""
+            },   
+        }
+
+        try:
+            mva_str = json.dumps(mva_results, indent=2, default=str)
+            final_result['missing_value_analysis']['report'] = self.generate_report_from_agent(mva_str)
+        except:
+            logger.error("Failed to generate report for mva....", log_type='data_quality_assessment', console=verbose)
+            pass
+
+        try:
+            duplicate_analysis_results_str = json.dumps(duplicate_analysis_results, indent=2, default=str)
+            final_result['duplicate_analysis']['report'] = self.generate_report_from_agent(duplicate_analysis_results_str)
+        except:
+            logger.error("Failed to generate report for duplicate analysis....", log_type='data_quality_assessment', console=verbose)
+            pass
+
+        try:
+            data_quality_results_str = json.dumps(data_quality_results, indent=2, default=str)
+            data_quality_results_str = data_quality_results_str +'\n'+data_quality_summary
+            final_result['data_quality_analysis']['report'] = self.generate_report_from_agent(data_quality_results_str)
+        except:
+            logger.error("Failed to generate report for data quality....", log_type='data_quality_assessment', console=verbose)
+            pass
+
+
+        return final_result

src/app/pipelines/modules/data_statistics.py

@@ -0,0 +1,1270 @@
+import os
+import re
+import math
+import json
+import psutil
+import numpy as np
+import pandas as pd
+import dateutil.parser
+from dotenv import load_dotenv
+from dateutil import parser
+from datetime import datetime
+from collections import Counter
+from src.core.utils import logger
+from agno.agent import Agent, RunResponse
+from agno.models.openai import OpenAIChat
+from typing import Union, List, Dict, Any, Tuple
+
+load_dotenv()
+
+class DataStatisticsWorkflow:
+    def __init__(
+        self, data_source: str,
+        llm_choice: str,
+        ml_task: str
+    ) -> None:
+        ''''''
+        self.data = None
+        self.llm = OpenAIChat(id=llm_choice, api_key=os.getenv('OPENAI_API_KEY'))
+        self.ml_task = ml_task
+        _ = self.load_data(data_source=data_source)
+        self.llm = OpenAIChat(id=llm_choice, api_key=os.getenv('OPENAI_API_KEY'))
+        self.writer: Agent = Agent(
+            model=self.llm,
+            instructions=[
+                "You will be provided with lots of structured outputs. Your work is to display this"
+                "in a nicely formatted manner. You must analayze the results and output a comprehensive and insightful report"
+            ],
+            markdown=True,
+        )
+
+    def load_data(self, data_source: str) -> Union[None, bool]:
+        '''Load CSV into dataframe'''
+        try:
+            self.data = pd.read_csv(data_source)
+            return True
+        except Exception as e:
+            logger.error(
+                f"Failed to read the file from the data source with error: {e}", log_type="data_statistics", console=True)
+            return False
+
+    def format_json(self, results, indent: int = 4) -> str:
+        def convert_to_serializable(obj):
+            if isinstance(obj, (np.integer, np.floating, np.bool_)):
+                return obj.item()
+            elif isinstance(obj, np.ndarray):
+                return obj.tolist()
+            elif pd.isna(obj):
+                return None
+            return obj
+
+        def process_dict(d):
+            result = {}
+            for k, v in d.items():
+                if isinstance(v, dict):
+                    result[k] = process_dict(v)
+                elif isinstance(v, list):
+                    result[k] = [convert_to_serializable(item) for item in v]
+                else:
+                    result[k] = convert_to_serializable(v)
+            return result
+
+        serializable_results = process_dict(results)
+
+        return json.dumps(serializable_results, indent=indent)
+
+    def build_statistical_summary(self, data_source: str = None, verbose=False) -> Dict[str, Any]:
+        '''Get the basic central tendancy, dispersion, quantiles, distinct values, frequency distributions and sparsity'''
+        logger.info(f"Starting statistical analysis", log_type="data_statistics", console=verbose)
+        if data_source:
+            status = self.load_data(data_source=data_source)
+            if not status:
+                logger.info("Failed to load data. Can't build statistical summary",
+                            log_type='data_statistics', console=True)
+
+        quantiles: List[float] = [0.05, 0.25, 0.5, 0.75, 0.95, 0.99]
+        num_freq_values: int = 10
+
+        results = {
+            "dataset_info": {
+                "num_rows": len(self.data),
+                "num_columns": len(self.data.columns),
+                "memory_usage": self.data.memory_usage(deep=True).sum(),
+                "dtypes": {col: str(dtype) for col, dtype in self.data.dtypes.items()}
+            },
+            "columns": {}
+        }
+
+        for column in self.data.columns:
+            column_data = self.data[column]
+            column_type = str(self.data[column].dtype)
+            is_numeric = pd.api.types.is_numeric_dtype(column_data)
+            is_datetime = pd.api.types.is_datetime64_any_dtype(column_data)
+
+            col_results = {
+                "dtype": column_type,
+                "count": len(column_data),
+                "num_unique": column_data.nunique(),
+                "num_missing": column_data.isna().sum(),
+                "missing_percentage": (column_data.isna().sum() / len(column_data)) * 100,
+                "sparsity": {
+                    "zeros": None,
+                    "zeros_percentage": None,
+                    "empty_strings": None,
+                    "empty_strings_percentage": None
+                }
+            }
+
+            is_boolean = pd.api.types.is_bool_dtype(column_data)
+
+            if is_numeric and not is_boolean:
+                try:
+                    num_zeros = (column_data == 0).sum()
+                    col_results["sparsity"]["zeros"] = num_zeros
+                    col_results["sparsity"]["zeros_percentage"] = (num_zeros / column_data.count()) * 100
+                except Exception as e:
+                    logger.error(f"{e}", log_type='data_statistics', console=verbose)
+            elif column_type.startswith('object') or column_type.startswith('string'):
+                try:
+                    empty_strings = column_data.fillna('').apply(lambda x: isinstance(x, str) and x.strip() == '').sum()
+                    col_results["sparsity"]["empty_strings"] = empty_strings
+                    col_results["sparsity"]["empty_strings_percentage"] = (empty_strings / column_data.count()) * 100
+                except Exception as e:
+                    logger.error(f"{e}", log_type='data_statistics', console=verbose)
+
+            if is_numeric and not is_boolean:
+                try:
+                    col_results["central_tendency"] = {
+                        "mean": column_data.mean() if not all(column_data.isna()) else None,
+                        "median": column_data.median() if not all(column_data.isna()) else None,
+                        "mode": column_data.mode().iloc[0] if not column_data.mode().empty else None
+                    }
+                except Exception as e:
+                    logger.error(f"{e}", log_type='data_statistics', console=verbose)
+
+                try:       
+                    col_results["dispersion"] = {
+                        "std": column_data.std() if not all(column_data.isna()) else None,
+                        "variance": column_data.var() if not all(column_data.isna()) else None,
+                        "range": {
+                            "min": column_data.min() if not all(column_data.isna()) else None,
+                            "max": column_data.max() if not all(column_data.isna()) else None
+                        },
+                       "iqr": (
+                            column_data.quantile(0.75) - column_data.quantile(0.25)
+                            if not pd.api.types.is_bool_dtype(column_data)
+                            else None
+                        )
+                    }
+                except Exception as e:
+                    logger.error(f"{e}", log_type='data_statistics', console=verbose)
+
+                if not all(column_data.isna()):
+                    col_results["quantiles"] = {
+                        f"q{int(q*100)}": column_data.quantile(q) for q in quantiles
+                    }
+                else:
+                    col_results["quantiles"] = {
+                        f"q{int(q*100)}": None for q in quantiles}
+
+                value_counts = column_data.value_counts(
+                    dropna=False).head(num_freq_values)
+                col_results["frequency_distribution"] = {
+                    "values": value_counts.index.tolist(),
+                    "counts": value_counts.tolist(),
+                    "percentages": (value_counts / len(column_data) * 100).tolist()
+                }
+
+                if is_datetime:
+                    try:
+                        col_results["datetime_info"] = {
+                            "min_date": column_data.min().strftime('%Y-%m-%d %H:%M:%S') if not all(column_data.isna()) else None,
+                            "max_date": column_data.max().strftime('%Y-%m-%d %H:%M:%S') if not all(column_data.isna()) else None,
+                            "date_range_days": (column_data.max() - column_data.min()).days if not all(column_data.isna()) else None
+                        }
+                    except:
+                        col_results["datetime_info"] = "Error processing datetime information"
+
+                results["columns"][column] = col_results
+
+        numeric_cols = self.data.select_dtypes(include=['number']).columns.tolist()
+        if len(numeric_cols) > 1:
+            try:
+                correlations = self.data[numeric_cols].corr().round(
+                    3).to_dict()
+                results["correlations"] = correlations
+            except:
+                results["correlations"] = "Error computing correlations"
+
+        return results
+
+    def analyze_data_types(self, sample_size=None, verbose=False):
+        logger.info(f"Starting data-type analysis", log_type="data_statistics", console=verbose)
+            
+        df = self.data
+        
+        if sample_size and sample_size < len(self.data):
+            df = self.data.sample(sample_size, random_state=42)
+        
+        df = df.replace('', np.nan)
+        
+        results = {}
+        
+        for column in df.columns:
+            if df[column].isna().all():
+                results[column] = {
+                    "inferred_type": "empty",
+                    "description": "Column is entirely empty"
+                }
+                continue
+            
+            values = df[column].dropna().values
+            if len(values) == 0:
+                continue
+                
+            column_analysis = self.analyze_column(values, column, df)
+            results[column] = column_analysis
+        
+        results["__summary__"] = self.generate_summary(results, df)
+        return results
+
+    def analyze_column(self, values, column_name, df):
+        """Analyze a single column's values to determine data types and patterns (safe and error-resilient)"""
+        try:
+            sample_values = values[:5].tolist() if hasattr(values, 'tolist') else list(values)[:5]
+        except Exception:
+            sample_values = []
+
+        type_counts = {
+            "integer": 0,
+            "float": 0,
+            "boolean": 0,
+            "date": 0,
+            "text": 0
+        }
+
+        unique_values = set()
+        total_values = len(values)
+        
+        for val in values:
+            try:
+                unique_values.add(val)
+                detected_type = self.detect_value_type(val)
+                if detected_type:
+                    type_counts[detected_type] += 1
+            except Exception:
+                continue
+
+        try:
+            main_type = max(type_counts.items(), key=lambda x: x[1])[0]
+            mixed_types = sum(1 for count in type_counts.values() if count > 0) > 1
+            type_percentages = {
+                t: (count / total_values) * 100 for t, count in type_counts.items() if count > 0
+            }
+        except Exception:
+            main_type = "text"
+            mixed_types = False
+            type_percentages = {}
+
+        result = {
+            "inferred_type": main_type,
+            "has_mixed_types": mixed_types,
+            "type_percentages": type_percentages,
+            "unique_count": len(unique_values),
+            "unique_percentage": (len(unique_values) / total_values * 100) if total_values else 0,
+            "sample_values": sample_values
+        }
+
+        try:
+            if main_type in ["integer", "float"]:
+                numeric_analysis = self.analyze_numeric_column(values, result)
+                result.update(numeric_analysis)
+        except Exception:
+            pass
+
+        try:
+            if main_type == "date":
+                date_analysis = self.analyze_date_column(values)
+                result.update(date_analysis)
+        except Exception:
+            pass
+
+        if main_type == "text":
+            try:
+                numeric_as_text = self.check_numeric_as_text(values)
+                if numeric_as_text.get("is_numeric_as_text"):
+                    result["numeric_as_text"] = True
+                    result["numeric_as_text_details"] = numeric_as_text
+            except Exception:
+                pass
+
+            try:
+                date_as_text = self.check_dates_as_text(values)
+                if date_as_text.get("is_date_as_text"):
+                    result["date_as_text"] = True
+                    result["date_as_text_details"] = date_as_text
+            except Exception:
+                pass
+
+        try:
+            cardinality = self.analyze_cardinality(values, result["unique_count"], total_values)
+            result.update(cardinality)
+        except Exception:
+            pass
+
+        try:
+            if result["unique_count"] <= 2 and not result.get("is_id_like", False):
+                result["is_binary"] = True
+                result["binary_values"] = list(unique_values)
+            else:
+                result["is_binary"] = False
+        except Exception:
+            result["is_binary"] = False
+
+        try:
+            result["description"] = self.generate_column_description(result, column_name)
+        except Exception:
+            result["description"] = "Could not generate column description due to an error."
+
+        return result
+
+    def detect_value_type(self, value, verbose=False):
+        """Determine the data type of a single value"""
+        try:
+            if value.lower() in ('true', 'false', 'yes', 'no', 'y', 'n', 't', 'f', '1', '0'):
+                if value.lower() in ('1', '0'):
+                    pass
+                else:
+                    return "boolean"
+            
+            if re.match(r'^-?\d+$', value):
+                return "integer"
+            
+            if re.match(r'^-?\d+\.\d+$', value) or re.match(r'^-?\d+,\d+$', value):
+                return "float"
+            
+            try:
+                dateutil.parser.parse(value)
+                return "date"
+            except:
+                pass
+            
+            return "text"
+        except Exception as e:
+            logger.error(f"Value type detection failed with error: {e}", log_type='data_statistics', console=verbose)
+            return ""
+
+    def analyze_date_column(self, values):
+        """Further analyze date columns safely and thoroughly"""
+        from datetime import datetime
+        formats = []
+        min_date = None
+        max_date = None
+
+        for val in values[:100]:  
+            if not val:
+                continue
+            try:
+                date_val = val if isinstance(val, datetime) else dateutil.parser.parse(str(val))
+
+                if not min_date or date_val < min_date:
+                    min_date = date_val
+                if not max_date or date_val > max_date:
+                    max_date = date_val
+
+                val_str = str(val).strip()
+                if re.match(r'^\d{4}-\d{2}-\d{2}$', val_str):
+                    formats.append("YYYY-MM-DD")
+                elif re.match(r'^\d{4}/\d{2}/\d{2}$', val_str):
+                    formats.append("YYYY/MM/DD")
+                elif re.match(r'^\d{2}-\d{2}-\d{4}$', val_str):
+                    formats.append("DD-MM-YYYY")
+                elif re.match(r'^\d{2}/\d{2}/\d{4}$', val_str):
+                    formats.append("DD/MM/YYYY") 
+                elif re.match(r'^\d{2}-\w{3}-\d{4}$', val_str):
+                    formats.append("DD-MMM-YYYY")
+                elif re.match(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$', val_str):
+                    formats.append("YYYY-MM-DD HH:MM:SS")
+                else:
+                    formats.append("Other")
+            except Exception:
+                continue
+
+        format_counts = Counter(formats)
+        most_common = format_counts.most_common(2)
+
+        result = {
+            "date_format": [f[0] for f in most_common] if most_common else ["Unknown"],
+            "min_date": min_date.isoformat() if min_date else None,
+            "max_date": max_date.isoformat() if max_date else None,
+            "date_range_days": (max_date - min_date).days if min_date and max_date else None,
+            "confidence": (len(formats) / min(len(values), 100)) * 100
+        }
+
+        return result
+
+    def analyze_numeric_column(self, values, current_results):
+        """Further analyze numeric columns (safe version)"""
+        result = {}
+
+        try:
+            numeric_values = []
+            for val in values:
+                try:
+                    if val is None:
+                        continue
+                    if isinstance(val, (int, float)):
+                        numeric_values.append(float(val))
+                    elif isinstance(val, str):
+                        cleaned = val.replace(',', '.').strip()
+                        numeric_values.append(float(cleaned))
+                except (ValueError, TypeError):
+                    continue
+
+            if not numeric_values:
+                result["warning"] = "No valid numeric values found"
+                return result
+
+            numeric_values.sort()
+            n = len(numeric_values)
+            median = (
+                numeric_values[n // 2]
+                if n % 2 == 1
+                else (numeric_values[n // 2 - 1] + numeric_values[n // 2]) / 2
+            )
+
+            result.update({
+                "min": min(numeric_values),
+                "max": max(numeric_values),
+                "mean": sum(numeric_values) / n,
+                "median": median
+            })
+
+            if current_results.get("inferred_type") == "float":
+                integers_count = sum(1 for x in numeric_values if x == int(x))
+                result["integers_percentage"] = (integers_count / n) * 100
+
+        except Exception as e:
+            result["error"] = f"Error in analyze_numeric_column: {str(e)}"
+
+        return result
+
+    def check_numeric_as_text(self, values):
+        """Check if text values might actually be numeric values stored as text (safe version)"""
+        result = {
+            "is_numeric_as_text": False,
+            "confidence": 0.0,
+            "matched_patterns": {}
+        }
+
+        try:
+            if not values:
+                result["warning"] = "No values provided"
+                return result
+
+            sample_size = min(100, len(values))
+            if sample_size == 0:
+                result["warning"] = "No valid sample size"
+                return result
+
+            numeric_patterns = [
+                r'^\d{1,3}(,\d{3})+(\.\d+)?$',   # US format: 1,234.56
+                r'^\d{1,3}(\.\d{3})+(,\d+)?$',   # EU format: 1.234,56
+                r'^-?\d+\s\d+(/\d+)?$',          # Fractions: 1 1/2
+                r'^\$\d+(\.\d+)?$',              # Dollars: $123.45
+                r'^€\d+(\.\d+)?$',               # Euros: €123.45
+                r'^\d+(\.\d+)?%$',               # Percentages: 12.3%
+            ]
+
+            matches = 0
+            pattern_matches = {pattern: 0 for pattern in numeric_patterns}
+
+            for val in values[:sample_size]:
+                try:
+                    if not isinstance(val, str):
+                        continue
+                    for pattern in numeric_patterns:
+                        if re.match(pattern, val.strip()):
+                            matches += 1
+                            pattern_matches[pattern] += 1
+                            break
+                except Exception:
+                    continue
+
+            percentage = (matches / sample_size) * 100
+            is_numeric_as_text = percentage > 80
+
+            result.update({
+                "is_numeric_as_text": is_numeric_as_text,
+                "confidence": percentage,
+                "matched_patterns": {p: c for p, c in pattern_matches.items() if c > 0}
+            })
+
+        except Exception as e:
+            result["error"] = f"Error in check_numeric_as_text: {str(e)}"
+
+        return result
+
+    def check_dates_as_text(self, values):
+        """Check if text values might actually be dates stored as text (safe version)"""
+        result = {
+            "is_date_as_text": False,
+            "confidence": 0.0,
+            "common_formats": []
+        }
+
+        try:
+            if not values:
+                result["warning"] = "No values provided"
+                return result
+
+            sample_size = min(100, len(values))
+            dates_detected = 0
+            formats_detected = []
+
+            for val in values[:sample_size]:
+                try:
+                    if not isinstance(val, str):
+                        continue
+
+                    parser.parse(val)
+                    dates_detected += 1
+
+                    if re.match(r'^\d{4}-\d{2}-\d{2}', val):
+                        formats_detected.append("ISO")
+                    elif re.match(r'^\d{2}/\d{2}/\d{4}', val):
+                        formats_detected.append("US/UK")
+                    elif re.match(r'^\d{2}-\w{3}-\d{4}', val):
+                        formats_detected.append("DD-MMM-YYYY")
+                    else:
+                        formats_detected.append("Other")
+
+                except Exception:
+                    continue
+
+            if sample_size > 0:
+                percentage = (dates_detected / sample_size) * 100
+            else:
+                percentage = 0.0
+
+            is_date_as_text = percentage > 80
+            format_counts = Counter(formats_detected)
+            common_formats = format_counts.most_common(2)
+
+            result.update({
+                "is_date_as_text": is_date_as_text,
+                "confidence": percentage,
+                "common_formats": common_formats if formats_detected else []
+            })
+
+        except Exception as e:
+            result["error"] = f"Error in check_dates_as_text: {str(e)}"
+
+        return result
+
+    def analyze_cardinality(self, values, unique_count, total_count):
+        """Analyze cardinality to determine if a column is categorical, continuous, or ID-like, safely"""
+        result = {}
+
+        try:
+            if not values or total_count == 0:
+                result["error"] = "Empty values or total_count is 0"
+                return result
+
+            uniqueness_ratio = unique_count / total_count if total_count else 0
+
+            try:
+                value_counts = Counter(values)
+                most_common = value_counts.most_common(5)
+                top_value_percentage = most_common[0][1] / total_count * 100 if most_common else 0
+            except Exception as e:
+                most_common = []
+                top_value_percentage = 0
+                result["warning"] = f"Error analyzing value counts: {str(e)}"
+
+            if uniqueness_ratio > 0.9:
+                result["is_id_like"] = True
+                result["cardinality_type"] = "id_like"
+            elif uniqueness_ratio > 0.5:
+                result["is_continuous"] = True
+                result["cardinality_type"] = "continuous"
+            else:
+                result["is_categorical"] = True
+                result["cardinality_type"] = "categorical"
+
+                if unique_count <= 5:
+                    result["categorical_level"] = "low"
+                elif unique_count <= 20:
+                    result["categorical_level"] = "medium"
+                else:
+                    result["categorical_level"] = "high"
+
+                result["top_categories"] = [
+                    {
+                        "value": val,
+                        "count": count,
+                        "percentage": (count / total_count * 100 if total_count else 0)
+                    }
+                    for val, count in most_common
+                ]
+
+            result["uniqueness_ratio"] = uniqueness_ratio
+            result["top_value_percentage"] = top_value_percentage
+
+        except Exception as e:
+            result["error"] = f"Error in analyze_cardinality: {str(e)}"
+
+        return result
+
+    def generate_column_description(self, analysis, column_name):
+        """Generate a human-readable description of the column based on analysis, with error safety"""
+        descriptions = []
+
+        try:
+            if analysis.get("has_mixed_types", False):
+                inferred = analysis.get("inferred_type", "unknown")
+                descriptions.append(f"Mixed data types with {inferred} being most common")
+            else:
+                inferred = analysis.get("inferred_type", "unknown")
+                descriptions.append(f"{inferred.capitalize()} data type")
+        except Exception as e:
+            descriptions.append(f"[Error identifying type]: {str(e)}")
+
+        try:
+            if analysis.get("is_id_like", False):
+                descriptions.append("Likely an ID field (high uniqueness)")
+            elif analysis.get("is_binary", False):
+                binary_values = ", ".join(str(v) for v in analysis.get("binary_values", []))
+                descriptions.append(f"Binary field with values: {binary_values}")
+            elif analysis.get("is_categorical", False):
+                cardinality = analysis.get("categorical_level", "unknown")
+                unique_count = analysis.get("unique_count", "unknown")
+                descriptions.append(f"{cardinality.capitalize()} cardinality categorical field with {unique_count} unique values")
+            elif analysis.get("is_continuous", False):
+                descriptions.append("Continuous variable")
+        except Exception as e:
+            descriptions.append(f"[Error determining field category]: {str(e)}")
+
+        try:
+            if analysis.get("numeric_as_text", False):
+                descriptions.append("Numeric values stored as text")
+        except Exception:
+            pass
+
+        try:
+            if analysis.get("date_as_text", False):
+                descriptions.append("Date values stored as text")
+        except Exception:
+            pass
+
+        try:
+            if "min" in analysis and "max" in analysis:
+                descriptions.append(f"Range: {analysis['min']} to {analysis['max']}")
+        except Exception as e:
+            descriptions.append(f"[Error extracting range]: {str(e)}")
+
+        try:
+            if "date_format" in analysis:
+                descriptions.append(f"Date format: {analysis['date_format']}")
+                if "date_range_days" in analysis:
+                    descriptions.append(f"Spans {analysis['date_range_days']} days")
+        except Exception as e:
+            descriptions.append(f"[Error in date format]: {str(e)}")
+
+        return ". ".join(descriptions)
+
+    def generate_summary(self, results, df):
+        """Generate an overall summary of the dataset with error safety"""
+        analysis_results = {k: v for k, v in results.items() if k != "__summary__"}
+        
+        column_types = {}
+        mixed_types_count = numeric_as_text_count = date_as_text_count = 0
+        binary_count = id_like_count = categorical_count = continuous_count = 0
+
+        for col, analysis in analysis_results.items():
+            try:
+                col_type = analysis.get("inferred_type", "unknown")
+                column_types[col_type] = column_types.get(col_type, 0) + 1
+            except Exception as e:
+                column_types["unknown"] = column_types.get("unknown", 0) + 1
+
+            try:
+                if analysis.get("has_mixed_types", False):
+                    mixed_types_count += 1
+            except Exception:
+                pass
+
+            try:
+                if analysis.get("numeric_as_text", False):
+                    numeric_as_text_count += 1
+            except Exception:
+                pass
+
+            try:
+                if analysis.get("date_as_text", False):
+                    date_as_text_count += 1
+            except Exception:
+                pass
+
+            try:
+                if analysis.get("is_binary", False):
+                    binary_count += 1
+            except Exception:
+                pass
+
+            try:
+                if analysis.get("is_id_like", False):
+                    id_like_count += 1
+            except Exception:
+                pass
+
+            try:
+                if analysis.get("is_categorical", False):
+                    categorical_count += 1
+            except Exception:
+                pass
+
+            try:
+                if analysis.get("is_continuous", False):
+                    continuous_count += 1
+            except Exception:
+                pass
+
+        recommendations = []
+        if mixed_types_count > 0:
+            recommendations.append(f"Review {mixed_types_count} columns with mixed data types")
+        if numeric_as_text_count > 0:
+            recommendations.append(f"Convert {numeric_as_text_count} columns with numeric values stored as text")
+        if date_as_text_count > 0:
+            recommendations.append(f"Convert {date_as_text_count} columns with date values stored as text")
+
+        try:
+            detailed_findings = self.generate_key_findings(analysis_results)
+        except Exception as e:
+            detailed_findings = [f"[Error generating key findings]: {str(e)}"]
+
+        return {
+            "total_rows": len(df),
+            "total_columns": len(analysis_results),
+            "column_type_counts": column_types,
+            "data_quality": {
+                "mixed_types_count": mixed_types_count,
+                "numeric_as_text_count": numeric_as_text_count,
+                "date_as_text_count": date_as_text_count
+            },
+            "structure": {
+                "binary_columns": binary_count,
+                "id_like_columns": id_like_count,
+                "categorical_columns": categorical_count,
+                "continuous_columns": continuous_count,
+            },
+            "recommendations": recommendations,
+            "detailed_findings": detailed_findings
+        }
+
+    def generate_key_findings(self, analysis_results):
+        """Generate detailed key findings from the analysis results with error safety"""
+        findings = []
+
+        for col, analysis in analysis_results.items():
+            try:
+                if analysis.get("has_mixed_types", False):
+                    type_percentages = analysis.get("type_percentages", {})
+                    finding = f"Column '{col}' has mixed data types: " + ", ".join(
+                        f"{type_name} ({percentage:.1f}%)"
+                        for type_name, percentage in type_percentages.items()
+                    )
+                    findings.append(finding)
+            except Exception as e:
+                findings.append(f"[Error analyzing mixed types for '{col}']: {e}")
+
+            try:
+                if analysis.get("numeric_as_text", False):
+                    details = analysis.get("numeric_as_text_details", {})
+                    confidence = details.get("confidence", 0.0)
+                    findings.append(
+                        f"Column '{col}' likely contains numeric values stored as text ({confidence:.1f}% confidence)"
+                    )
+            except Exception as e:
+                findings.append(f"[Error analyzing numeric-as-text for '{col}']: {e}")
+
+            try:
+                if analysis.get("date_as_text", False):
+                    details = analysis.get("date_as_text_details", {})
+                    confidence = details.get("confidence", 0.0)
+                    formats = details.get("common_formats", [])
+                    format_str = ", ".join(f"{fmt[0]}" for fmt in formats[:2]) if formats else "various"
+                    findings.append(
+                        f"Column '{col}' likely contains dates in {format_str} format ({confidence:.1f}% confidence)"
+                    )
+            except Exception as e:
+                findings.append(f"[Error analyzing date-as-text for '{col}']: {e}")
+
+            try:
+                if analysis.get("is_id_like", False):
+                    uniqueness_ratio = analysis.get("uniqueness_ratio", 1.0)
+                    if uniqueness_ratio < 1.0:
+                        dupe_percentage = (1 - uniqueness_ratio) * 100
+                        findings.append(
+                            f"ID-like column '{col}' has {dupe_percentage:.1f}% duplicate values"
+                        )
+            except Exception as e:
+                findings.append(f"[Error analyzing ID-like column '{col}']: {e}")
+
+        return findings
+
+    def analyze_dataset_dimensionality(self, sample_size=None, detailed=True, verbose=False):
+        """Analyze the dimensionality characteristics of a dataset"""
+        logger.info(f"Starting dataset dimensionality analysis", log_type="data_statistics", console=verbose)
+
+        results = {
+            "basic_info": {},
+            "aspect_ratio": {},
+            "sparsity": {},
+            "memory_usage": {},
+            "processing_complexity": {},
+            "timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        }
+        
+        analysis_failures = []
+
+        try:
+            df = self.data
+            
+            if sample_size and sample_size < len(df):
+                df = df.sample(sample_size, random_state=42)
+            
+            results["basic_info"] = {
+                "rows": len(df),
+                "columns": len(df.columns),
+                "total_cells": len(df) * len(df.columns),
+                "column_names": list(df.columns)
+            }
+        except Exception as e:
+            results["basic_info"] = {
+                "status": "failed",
+                "error": str(e)
+            }
+            analysis_failures.append("basic_info")
+
+            results["status"] = "failed"
+            results["error"] = f"Failed to read CSV file: {str(e)}"
+            return results
+        
+       
+
+        if sample_size and sample_size < len(df):
+            df = df.sample(sample_size, random_state=42)
+        
+        try:
+            rows = results["basic_info"]["rows"]
+            cols = results["basic_info"]["columns"]
+            
+            results["aspect_ratio"] = {
+                "rows_to_columns_ratio": rows / cols if cols > 0 else float('inf'),
+                "columns_to_rows_ratio": cols / rows if rows > 0 else float('inf'),
+                "is_wide": cols > rows,
+                "is_tall": rows > cols,
+                "shape_description": self.get_shape_description(rows, cols)
+            }
+        except Exception as e:
+            results["aspect_ratio"] = {
+                "status": "failed",
+                "error": str(e)
+            }
+            analysis_failures.append("aspect_ratio")
+        
+        try:
+            null_count = df.isna().sum().sum()
+            total_cells = results["basic_info"]["total_cells"]
+            
+            empty_str_count = 0
+            for col in df.select_dtypes(include=['object']).columns:
+                try:
+                    empty_str_count += (df[col] == "").sum()
+                except:
+                    pass  
+            
+            zero_count = 0
+            for col in df.select_dtypes(include=['number']).columns:
+                try:
+                    zero_count += (df[col] == 0).sum()
+                except:
+                    pass  
+
+            null_sparsity = null_count / total_cells if total_cells > 0 else 0
+            total_sparsity = (null_count + empty_str_count) / total_cells if total_cells > 0 else 0
+            
+            results["sparsity"] = {
+                "null_count": int(null_count),
+                "empty_string_count": int(empty_str_count),
+                "zero_count_in_numeric": int(zero_count),
+                "null_sparsity": null_sparsity,
+                "total_sparsity": total_sparsity,
+                "sparsity_percentage": total_sparsity * 100,
+                "sparsity_level": self.get_sparsity_level(total_sparsity)
+            }
+            
+            if detailed:
+                col_sparsity = {}
+                for col in df.columns:
+                    null_pct = df[col].isna().mean() * 100
+                    
+                    empty_pct = 0
+                    if df[col].dtype == 'object':
+                        try:
+                            empty_pct = (df[col] == "").mean() * 100
+                        except:
+                            pass
+                    
+                    col_sparsity[col] = {
+                        "null_percentage": null_pct,
+                        "empty_percentage": empty_pct,
+                        "total_sparsity": null_pct + empty_pct
+                    }
+                results["sparsity"]["column_sparsity"] = col_sparsity
+        except Exception as e:
+            results["sparsity"] = {
+                "status": "failed",
+                "error": str(e)
+            }
+            analysis_failures.append("sparsity")
+        
+        try:
+            mem_usage = df.memory_usage(deep=True)
+            total_mem = mem_usage.sum()
+            
+            mem_by_type = {}
+            for dtype in df.dtypes.value_counts().index:
+                dtype_cols = df.select_dtypes(include=[dtype]).columns
+                dtype_mem = sum(mem_usage[col] for col in dtype_cols if col in mem_usage)
+                mem_by_type[str(dtype)] = dtype_mem
+            
+            full_dataset_mem = total_mem
+            if sample_size and sample_size < results["basic_info"]["rows"]:
+                full_rows = results["basic_info"]["rows"]
+                full_dataset_mem = total_mem * (full_rows / sample_size)
+            
+            try:
+                system_mem = psutil.virtual_memory().total
+            except:
+                system_mem = None
+            
+            results["memory_usage"] = {
+                "total_bytes": int(total_mem),
+                "total_mb": round(total_mem / (1024**2), 2),
+                "memory_by_type": {k: int(v) for k, v in mem_by_type.items()},
+                "average_bytes_per_row": int(total_mem / len(df)) if len(df) > 0 else 0,
+                "full_dataset_estimate_mb": round(full_dataset_mem / (1024**2), 2),
+                "system_memory_gb": round(system_mem / (1024**3), 2) if system_mem else None,
+                "memory_utilization_percentage": round(full_dataset_mem / system_mem * 100, 2) if system_mem else None
+            }
+        except Exception as e:
+            results["memory_usage"] = {
+                "status": "failed",
+                "error": str(e)
+            }
+            analysis_failures.append("memory_usage")
+        
+        try:
+            rows = results["basic_info"]["rows"]
+            cols = results["basic_info"]["columns"]
+            
+            basic_complexity = rows * cols  # O(n*m) operations
+            sorting_complexity = rows * math.log2(rows) if rows > 0 else 0  # O(n log n)
+            
+            unique_counts = {}
+            for col in df.select_dtypes(include=['object']).columns:
+                try:
+                    unique_counts[col] = df[col].nunique()
+                except:
+                    pass
+            
+            num_numeric = len(df.select_dtypes(include=['number']).columns)
+            
+            complexity_level = self.get_complexity_level(rows, cols)
+            
+            results["processing_complexity"] = {
+                "basic_complexity": basic_complexity,
+                "sorting_complexity": sorting_complexity,
+                "unique_values_in_string_columns": unique_counts,
+                "numeric_columns_count": num_numeric,
+                "complexity_level": complexity_level,
+                "processing_recommendation": self.get_processing_recommendation(rows, cols)
+            }
+        except Exception as e:
+            results["processing_complexity"] = {
+                "status": "failed",
+                "error": str(e)
+            }
+            analysis_failures.append("processing_complexity")
+        
+        try:
+            results["summary"] = self.generate_dimensionality_summary(results, analysis_failures)
+        except Exception as e:
+            results["summary"] = {
+                "status": "failed",
+                "error": str(e)
+            }
+        
+        if analysis_failures:
+            results["status"] = "partial_success"
+            results["failed_analyses"] = analysis_failures
+        else:
+            results["status"] = "success"
+        
+        return results
+
+    def get_shape_description(self, rows, cols):
+        """Get a descriptive term for the dataset shape based on rows and columns ratio"""
+        ratio = rows / cols if cols > 0 else float('inf')
+        
+        if ratio > 1000:
+            return "extremely_tall"
+        elif ratio > 100:
+            return "very_tall"
+        elif ratio > 10:
+            return "tall"
+        elif ratio > 3:
+            return "moderately_tall"
+        elif ratio > 0.33:
+            return "balanced"
+        elif ratio > 0.1:
+            return "moderately_wide"
+        elif ratio > 0.01:
+            return "wide"
+        elif ratio > 0.001:
+            return "very_wide"
+        else:
+            return "extremely_wide"
+
+    def get_sparsity_level(self, sparsity_ratio):
+        """Get a descriptive term for the dataset sparsity level"""
+        if sparsity_ratio > 0.9:
+            return "extremely_sparse"
+        elif sparsity_ratio > 0.7:
+            return "very_sparse"
+        elif sparsity_ratio > 0.5:
+            return "sparse"
+        elif sparsity_ratio > 0.3:
+            return "moderately_sparse"
+        elif sparsity_ratio > 0.1:
+            return "slightly_sparse"
+        else:
+            return "dense"
+
+    def get_complexity_level(self, rows, cols):
+        """Get a descriptive term for processing complexity based on dataset size"""
+        cells = rows * cols
+        
+        if cells > 1_000_000_000:  # 1 billion cells
+            return "extremely_high"
+        elif cells > 100_000_000:  # 100 million cells
+            return "very_high"
+        elif cells > 10_000_000:  # 10 million cells
+            return "high"
+        elif cells > 1_000_000:  # 1 million cells
+            return "moderate"
+        elif cells > 100_000:  # 100,000 cells
+            return "low"
+        else:
+            return "very_low"
+
+    def get_processing_recommendation(self, rows, cols):
+        """Get processing recommendations based on dataset size"""
+        cells = rows * cols
+        
+        if cells > 1_000_000_000:  # 1 billion cells
+            return "Distributed computing recommended (Spark, Dask). Consider data sampling or partitioning."
+        elif cells > 100_000_000:  # 100 million cells
+            return "Optimized libraries recommended. Consider chunking data or using out-of-core processing."
+        elif cells > 10_000_000:  # 10 million cells
+            return "Standard pandas may be slow. Consider optimizing memory usage or using more efficient libraries."
+        elif cells > 1_000_000:  # 1 million cells
+            return "Standard pandas should work well with sufficient memory."
+        else:
+            return "Dataset can be easily processed with standard tools."
+
+    def generate_dimensionality_summary(self, results, analysis_failures):
+        """Generate a summary of the dimensionality analysis"""
+        summary = {}
+        
+        try:
+            rows = results["basic_info"]["rows"]
+            cols = results["basic_info"]["columns"]
+            shape_desc = results["aspect_ratio"]["shape_description"]
+            summary["shape"] = f"Dataset has {rows:,} rows and {cols:,} columns ({shape_desc} shape)"
+        except:
+            summary["shape"] = "Could not determine dataset shape"
+        
+        try:
+            null_count = results["sparsity"]["null_count"]
+            sparsity_pct = results["sparsity"]["sparsity_percentage"]
+            sparsity_level = results["sparsity"]["sparsity_level"]
+            summary["sparsity"] = f"Dataset is {sparsity_level} with {sparsity_pct:.1f}% missing values"
+        except:
+            summary["sparsity"] = "Could not determine dataset sparsity"
+        
+        try:
+            mem_mb = results["memory_usage"]["total_mb"]
+            full_est_mb = results["memory_usage"]["full_dataset_estimate_mb"]
+            if mem_mb == full_est_mb:
+                summary["memory"] = f"Memory usage: {mem_mb:.1f} MB"
+            else:
+                summary["memory"] = f"Memory usage: {mem_mb:.1f} MB (estimated full dataset: {full_est_mb:.1f} MB)"
+        except:
+            summary["memory"] = "Could not determine memory usage"
+        
+        try:
+            complexity = results["processing_complexity"]["complexity_level"]
+            summary["complexity"] = f"Processing complexity: {complexity}"
+        except:
+            summary["complexity"] = "Could not determine processing complexity"
+        
+        insights = []
+        
+        try:
+            if results["aspect_ratio"]["is_wide"]:
+                insights.append("Dataset has more columns than rows, which is unusual and may indicate a wide/transposed format")
+            
+            if cols > 100:
+                insights.append(f"High number of columns ({cols}) may indicate a need for dimensionality reduction")
+        except:
+            pass
+        
+        try:
+            if results["sparsity"]["sparsity_percentage"] > 50:
+                insights.append(f"Dataset is quite sparse ({results['sparsity']['sparsity_percentage']:.1f}% missing values), consider handling missing data")
+            
+            if "column_sparsity" in results["sparsity"]:
+                sparse_cols = [col for col, data in results["sparsity"]["column_sparsity"].items() 
+                            if data["total_sparsity"] > 80]
+                if sparse_cols:
+                    col_count = len(sparse_cols)
+                    if col_count <= 3:
+                        insights.append(f"Consider dropping or imputing very sparse columns: {', '.join(sparse_cols)}")
+                    else:
+                        insights.append(f"{col_count} columns have >80% missing values and may be candidates for removal")
+        except:
+            pass
+        
+        try:
+            if results["memory_usage"].get("memory_utilization_percentage", 0) > 50:
+                insights.append("Dataset is using a significant portion of system memory, consider chunking or sampling")
+                
+            if "memory_by_type" in results["memory_usage"]:
+                mem_by_type = results["memory_usage"]["memory_by_type"]
+                if "object" in mem_by_type:
+                    object_mem_ratio = mem_by_type["object"] / results["memory_usage"]["total_bytes"]
+                    if object_mem_ratio > 0.7:
+                        insights.append("String/object columns use most memory; consider category type conversion")
+        except:
+            pass
+        
+        try:
+            complexity_level = results["processing_complexity"]["complexity_level"]
+            if complexity_level in ["high", "very_high", "extremely_high"]:
+                insights.append(results["processing_complexity"]["processing_recommendation"])
+        except:
+            pass
+        
+        summary["key_insights"] = insights if insights else ["No specific insights identified"]
+        
+        recommendations = []
+        
+        try:
+            if cols > 100:
+                recommendations.append("Consider dimensionality reduction (PCA, feature selection)")
+            
+            if results["aspect_ratio"]["is_wide"]:
+                recommendations.append("Check if data is in correct format (might need transposing)")
+        except:
+            pass
+        
+        try:
+            if results["sparsity"]["sparsity_percentage"] > 30:
+                recommendations.append("Implement appropriate missing value strategy (imputation or removal)")
+        except:
+            pass
+        
+        try:
+            if "memory_by_type" in results["memory_usage"] and "object" in results["memory_usage"]["memory_by_type"]:
+                object_mem_ratio = results["memory_usage"]["memory_by_type"]["object"] / results["memory_usage"]["total_bytes"]
+                if object_mem_ratio > 0.5:
+                    recommendations.append("Optimize memory by converting string columns to categories or using more efficient data types")
+        except:
+            pass
+        
+        try:
+            complexity_level = results["processing_complexity"]["complexity_level"]
+            if complexity_level in ["very_high", "extremely_high"]:
+                recommendations.append("Use chunking, sampling or distributed processing for this dataset")
+        except:
+            pass
+        
+        summary["recommendations"] = recommendations if recommendations else ["No specific recommendations"]
+        
+        if analysis_failures:
+            summary["failures"] = f"Some analyses failed: {', '.join(analysis_failures)}"
+        
+        return summary
+
+    def generate_report_from_agent(self, input)->str:
+        '''Transform the json output to a user-readable report'''
+        try:
+            input = f"ML Task: {self.ml_task}\n{input}"
+            response: RunResponse = self.writer.run(input, stream=False)
+            return response.content
+        except Exception as e:
+            return f"Failed to generate report with error: {e}"
+
+    def convert_numpy_types(self, obj):
+        if isinstance(obj, dict):
+            return {k: self.convert_numpy_types(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [self.convert_numpy_types(item) for item in obj]
+        elif isinstance(obj, np.integer):
+            return int(obj)
+        elif isinstance(obj, np.floating):
+            return float(obj)
+        elif isinstance(obj, np.bool_):
+            return bool(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        else:
+            return obj
+
+    def run(self, verbose=False)-> Dict[str, dict]:
+        '''Run the entire workflow'''
+        statistical_summary_dict = self.build_statistical_summary(verbose=verbose)
+        data_type_analysis_dict = self.analyze_data_types(verbose=verbose)
+        dataset_dimensionality_dict = self.analyze_dataset_dimensionality(verbose=verbose)
+
+        statistical_summary_str = self.format_json(statistical_summary_dict)
+        data_type_analysis_str = self.format_json(data_type_analysis_dict)
+        dataset_dimensionality_str = self.format_json(dataset_dimensionality_dict)
+
+        logger.info("Generating final reports....", log_type='data_statistics', console=verbose)
+        statistical_report = self.generate_report_from_agent(input=statistical_summary_str)
+        data_type_report = self.generate_report_from_agent(input=data_type_analysis_str)
+        dataset_dimensionality_report = self.generate_report_from_agent(input=dataset_dimensionality_str)
+
+        final_result = {
+            "statistical_analysis": {
+                'dict': self.convert_numpy_types(statistical_summary_dict),
+                'report': statistical_report
+            },
+            "data_type_analysis": {
+                'dict': self.convert_numpy_types(data_type_analysis_dict),
+                'report': data_type_report
+            },
+            "dataset_dimensionality_analysis": {
+                'dict': self.convert_numpy_types(dataset_dimensionality_dict),
+                'report': dataset_dimensionality_report
+            },
+        }
+
+        return final_result
+
+        
+

src/app/pipelines/modules/data_understanding_context.py

@@ -0,0 +1,332 @@
+import os
+import random
+import pandas as pd
+from dotenv import load_dotenv
+from src.core.utils import logger
+from pydantic import BaseModel, Field
+from agno.models.openai import OpenAIChat
+from agno.agent import Agent, RunResponse
+from typing import Optional, Union, List, Dict, Tuple
+from agno.tools.duckduckgo import DuckDuckGoTools
+
+load_dotenv()
+
+# class BCAgentResponseSchema(BaseModel):
+#     executive_summary: Optional[str] = Field(
+#         default=None,
+#         description="Contains information about the industry recognized, business purpose, key insights and impact on analysis"
+#     )
+#     business_concept_mapping: Optional[str] = Field(
+#         default=None,
+#         description="Contains all the columns along with their business concept, definition, industry standard term and their metric"
+#     )
+#     business_dependencies: Optional[str] = Field(
+#         default=None,
+#         description="Contains all the columns along with their dependent variable(s), nature of dependency and business impact"
+#     )
+#     key_performance_indicators: Optional[str] = Field(
+#         default=None,
+#         description="Contains names of all the kpis along with their formula, business objective, target range and industry benchmark"
+#     )
+#     regulatory_compliance_considerations: Optional[List[str]] = Field(
+#         default=None,
+#         description="Contains list all the regulations along with their applicability, key requirements and affected variables"
+#     )
+#     business_rules: Optional[List[str]] = Field(
+#         default=None,
+#         description="Contains list of all the rules along with their description, affected variables, priority level"
+#     )
+#     impact_analysis: Optional[List[str]] = Field(
+#         default=None,
+#         description="Contains list of all the business context elements along with their data quality impact, analysis approach impact, modeling approach followed by their recommended approach"
+#     )
+#     references: Optional[List[str]] = Field(
+#         default=None,
+#         description="Contains the links for all the references and sources"
+#     )
+
+# class SUAgentResponseSchema(BaseModel):
+#     variable_definitions: Optional[Dict[str, str]] = Field(
+#         default=None,
+#         description="Contains the concise definition for each variable along with its unit of measurement observed"
+#     )
+#     reference_codes: Optional[Dict[str, str]] = Field(
+#         default=None,
+#         description="Contains the meaning for various abbreviations"
+#     )
+#     heirarchical_relationships: Optional[Dict[str, str]] = Field(
+#         default=None,
+#         description="Contains some information about heirarchy between variables"
+#     )
+
+
+# DATA SOURCE ASSESSMENT STILL INCOMPLETE
+
+class DataUnderstandingContextWorkflow:
+    def __init__(
+        self, data_source: str,
+        llm_choice: str,
+        source_context: Union[str, None] = None,
+        business_context: Union[str, None] = None
+    ) -> None:
+        '''This is the very 1st module which will be executed for all EDA tasks regardless'''
+
+        self.data = None
+        self.source_context = source_context
+        self.business_context = business_context
+        self.llm = OpenAIChat(
+            id=llm_choice, api_key=os.getenv('OPENAI_API_KEY'))
+
+        try:
+            self.data = pd.read_csv(data_source)
+        except Exception as e:
+            logger.error(
+                f"Failed to read the file from the data source with error: {e}", log_type="data_understanding_context", console=True)
+
+    def extract_column_data(self, max_col_limit=15) -> Union[str, None]:
+        """
+        Extracts column names and samples rows from a dataset.
+
+        Rules:
+        - If the dataset has more than 15 columns, sample 1 random row.
+        - Otherwise, sample 3 random rows.
+
+        Returns:
+            A formatted string containing column names and sampled data.
+        """
+
+        columns = self.data.columns.tolist()
+        num_columns = len(columns)
+
+        if num_columns > max_col_limit:
+            sampled_rows = self.data.sample(
+                1, random_state=random.randint(0, 1000))
+        else:
+            sampled_rows = self.data.sample(
+                3, random_state=random.randint(0, 1000))
+
+        doc = f"""
+        Dataset Summary
+        ===============
+
+        Total Columns: {num_columns}
+        Column Names:
+        {columns}
+
+        Sampled Data:
+        {sampled_rows.to_string(index=False)}
+        """
+
+        return doc.strip()
+
+    def build_business_context_integration(self, verbose=False) -> Union[RunResponse, None]:
+        logger.info(f"Starting business context integration", log_type='data_understanding_context', console=verbose)
+        if not self.business_context:
+            logger.info("No business context found. Skipping this section...",
+                        log_type='data_understanding_context', console=verbose)
+            return None
+        else:
+
+            try:
+                bc_agent = Agent(
+                    name="Businesss Context Agent",
+                    model=self.llm,
+                    markdown=True,
+                    reasoning=True,
+                    # response_model=BCAgentResponseSchema,
+                    instructions="""
+                        Business Context Integration Agent Prompt
+
+                        Your objective is to serve as a Business Context Integration Agent. Given a dataset with column names and the type of machine learning task (e.g., classification, regression, clustering), your responsibility is to gather authoritative, relevant business context by performing extensive web-based research.
+
+                         All claims must be supported with referenced links.
+
+                        Use the dataset columns and ML task as the anchor to drive your research and reasoning.
+
+                        -----------------------------------------------
+                        Research Goals:
+                        -----------------------------------------------
+
+                        1. executive_summary:
+                            - Identify the broader industry and functional domain that the dataset likely belongs to based on the column names.
+                            - Outline the business purpose for building the machine learning model.
+                            - Explain how incorporating domain knowledge impacts the model’s relevance and usefulness in a real-world context.
+                            - Highlight notable trends, challenges, or insights from the industry that are relevant to the problem space.
+
+                        2. business_concept_mapping:
+                            - Map each dataset column to its corresponding business concept and provide a clear and formal definition.
+                            - Identify industry-recognized terms and naming conventions related to the variables.
+                            - Include units of measurement, and where applicable, mathematical formulas or logic behind computed values.
+                            - Use standardized terminologies from domain-specific glossaries or data dictionaries to ensure consistency.
+
+                        3. business_dependencies:
+                            - For each variable, identify direct and indirect relationships with other variables in the dataset.
+                            - Define whether these relationships are statistical, causal, temporal, or logical.
+                            - Reference literature or domain analyses that justify these dependencies and describe their business implications.
+                            - Explain how these dependencies may affect downstream model features or interpretation.
+
+                        4. key_performance_indicators:
+                            - Determine the key performance indicators (KPIs) that are either directly present in the dataset or can be derived from it.
+                            - Provide detailed formulas, objectives behind each KPI, and their role in strategic or operational decision-making.
+                            - Include target thresholds or performance ranges based on benchmarks from industry or regulatory sources.
+                            - Document frameworks or methodologies used to calculate and track these KPIs.
+
+                        5. regulatory_compliance_considerations:
+                            - Identify regulations, compliance requirements, or legal constraints that apply to the variables or their use in machine learning models.
+                            - Summarize mandates from regulatory bodies, compliance frameworks, and data protection laws relevant to the domain.
+                            - Detail how these regulations affect data collection, storage, analysis, and reporting for the columns involved.
+                            - Note any enforcement actions or penalties associated with non-compliance, if applicable.
+
+                        6. business_rules:
+                            - Collect formal and informal business rules that define the valid behavior, constraints, or validation criteria for each variable.
+                            - Document rule sources such as process documentation, system specifications, industry best practices, and technical standards.
+                            - Classify rules by type (validation, constraint, workflow, consistency) and specify affected variables.
+                            - Assign a priority level (e.g., High, Medium, Low) to each rule based on its impact or frequency.
+
+                        7. impact_analysis:
+                            - Assess the influence of the business context elements (e.g., dependencies, rules, KPIs, regulations) on data quality and model effectiveness.
+                            - Highlight how they should inform exploratory analysis, feature selection, or modeling strategies.
+                            - Recommend specific techniques or precautions to address potential risks or quality issues stemming from business constraints or misalignments.
+                            - Discuss how different business-driven assumptions or constraints might affect model interpretability, performance, and deployment.
+
+                        8. references:
+                            - Provide a list of URLs for all sources consulted or referenced in each of the above categories.
+                            - Prioritize credible references including academic publications, regulatory websites, vendor documentation, whitepapers, and official data standards.
+
+                            
+                        9. metadata_analysis:
+                            - Evaluate the naming conventions of each column in the dataset to determine if they follow consistent, domain-aligned patterns.
+                                • Identify the naming style (e.g., snake_case, camelCase, abbreviations) and flag inconsistencies.
+                                • Recommend improvements to align with industry-standard naming practices where needed.
+                                • Justify suggestions based on business readability, clarity, and traceability.
+                            - Categorize all dataset variables into the following types:
+                                • Identifier Variables: Used to uniquely identify records (e.g., IDs, Ticket Numbers).
+                                • Analytical Variables: Variables that contain information relevant to prediction, segmentation, or business insight generation.
+                                • Target Variables: Outcome labels used in supervised machine learning tasks.
+                                • Derived or Redundant Variables: Columns generated from transformations, combinations, or duplicates of others.
+                            - Explain the business role and implications of each variable type:
+                                • Why are certain fields considered identifiers and not used for analysis?
+                                • Which analytical variables are central to decision-making and why?
+                                • How does identifying target and redundant variables support data quality and model interpretability?
+                            - Where applicable, reference industry data dictionaries or data modeling standards to support naming and classification.
+                        
+                        -----------------------------------------------
+                        Guidelines:
+                        -----------------------------------------------
+
+                        - Maintain a descriptive, business-friendly tone in all outputs.
+                        - Avoid technical jargon unless it is relevant to business understanding or regulatory compliance.
+                        - Do not make assumptions without a verifiable source; indicate low-confidence insights where applicable.
+                        - Ensure all responses are grounded in real-world context and supported by credible, traceable information.
+                        - Be comprehensive but concise—aim to deliver practical business value with each section of the output.
+                    """,
+                    tools=[DuckDuckGoTools(search=True, news=False)]
+                )
+
+                columns_data = self.extract_column_data()
+                prompt = (
+                    f'Business Context: \n'
+                    f'{self.business_context}\n'
+                    f'{columns_data}'
+                )
+
+                response: RunResponse = bc_agent.run(prompt, stream=False)
+
+                logger.info(f"Business Context integration finished....",
+                            log_type="data_understanding_context", console=verbose)
+                return response
+            except Exception as e:
+                logger.error(
+                    f"Failed to build business context integration with error: {e}", log_type="data_understanding_context", console=verbose)
+                return None
+
+    def build_semantic_understanding(self, verbose=False) -> Union[RunResponse, None]:
+        logger.info(f"Starting to build semantic integration", log_type='data_understanding_context', console=verbose)
+        try:
+            bc_agent = Agent(
+                name="Semantic Understanding Agent",
+                model=self.llm,
+                markdown=True,
+                reasoning=True,
+                # response_model=SUAgentResponseSchema,
+                instructions="""
+                    Objective:
+                    You are a Semantic Understanding Agent responsible for analyzing a dataset with no prior documentation. Your goal is to enrich each variable (column) by extracting and constructing meaningful metadata that helps clarify the variable’s semantic role in the dataset. 
+                    This process is essential for enabling downstream tasks like data validation, feature engineering, explainability, and regulatory compliance.
+
+                    You must generate detailed descriptions and metadata across the following four components for each column in the dataset:
+
+                    -------------------------------------------------------------
+                    1. Create/Validate Variable Definitions
+                    -------------------------------------------------------------
+                    - Infer and construct a clear, concise, and human-readable definition for each variable based solely on the column name, data type, and observed value patterns.
+                    - Where the column contains obvious domain terms, use domain-specific language to define them precisely.
+                    - If a variable is ambiguous, document your reasoning and note the uncertainty.
+                    - Ensure that the definition is useful for a business analyst or data scientist trying to understand the purpose of the variable.
+
+                    -------------------------------------------------------------
+                    2. Document Units of Measurement for Each Variable
+                    -------------------------------------------------------------
+                    - Determine if the variable has an implicit or explicit unit of measurement.
+                    - Use value ranges and patterns to infer the unit (e.g., numeric age values imply years).
+                    - If no unit is applicable (e.g., categorical strings), indicate “N/A”.
+                    - Standardize the unit using singular form and SI/non-SI conventions when applicable.
+
+                    -------------------------------------------------------------
+                    3. Identify Reference Codes and Their Meanings
+                    -------------------------------------------------------------
+                    - Identify variables that represent coded values, whether single-letter, numeric, or short abbreviations.
+                    - For such variables, list all unique values and attempt to map each one to a meaningful label.
+                    - Use statistical reasoning (value distribution) and common domain knowledge to deduce what each code stands for.
+                    - If meanings are uncertain, flag them with a confidence level.
+
+
+                    -------------------------------------------------------------
+                    4. Document Any Known Hierarchical Relationships
+                    -------------------------------------------------------------
+                    - Identify columns that relate to one another hierarchically, structurally, or contextually.
+                    - Describe the hierarchy and explain the nature of the relationship (e.g., parent-child, group-subgroup, categorical-numerical link).
+                    - If patterns suggest derived or grouped relationships, explain your logic.
+
+                    -------------------------------------------------------------
+                    Output Guidelines:
+                    -------------------------------------------------------------
+                    - Be precise and neutral in language.
+                    - If uncertain, include a note explaining ambiguity or assumption.
+                    - Avoid inventing information—only reason from what's in the dataset.
+
+                    -------------------------------------------------------------
+                    Final Note:
+                    -------------------------------------------------------------
+                    You do not have access to external documentation or glossaries. All your outputs must be reasoned and extracted using the dataset column names and data types alone
+                    """
+            )
+
+            columns_data = self.extract_column_data()
+
+            prompt = (
+                f"{columns_data}"
+            )
+
+            response: RunResponse = bc_agent.run(prompt, stream=False)
+
+            logger.info(f"Semantic Understanding Finished....",
+                        log_type="data_understanding_context", console=verbose)
+            return response
+        except Exception as e:
+            logger.error(
+                f"Failed to build Semantic Understanding with error: {e}", log_type="data_understanding_context", console=verbose)
+            return None
+
+    def run(self, verbose=False) -> Dict[str, dict]:
+        bci_result = self.build_business_context_integration(verbose=verbose)
+        su_result = self.build_semantic_understanding(verbose=verbose)
+
+        return {
+            'business_context_integration':{
+                "report": bci_result.content
+            },
+            'semantic_understanding': {
+                "report": su_result.content
+            }
+        }

src/app/pipelines/modules/univariate_analysis.py

@@ -0,0 +1,1437 @@
+import os
+import math
+import json
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+import lmoments3 as lm
+import scipy.stats as stats
+from src.core.utils import logger
+from agno.models.openai import OpenAIChat
+from agno.agent import Agent, RunResponse
+from sklearn.neighbors import KernelDensity
+from sklearn.model_selection import GridSearchCV
+from typing import Union, Tuple, Dict, Any, List, Optional
+from scipy.stats import norm, shapiro, anderson, kstest, normaltest
+
+class UnivariateAnalysisWorkflow:
+    def __init__(
+        self, data_source: str,
+        llm_choice: str,
+        ml_task: str
+    ) -> None:
+        ''''''
+        self.data = None
+        self.data_source = data_source
+        self.llm_choice = llm_choice
+        self.llm = OpenAIChat(id=llm_choice, api_key=os.getenv('OPENAI_API_KEY'))
+        self.writer: Agent = Agent(
+            model=self.llm,
+            instructions=[
+                "You will be provided with lots of structured outputs. Your work is to display this"
+                "in a nicely formatted manner. You must analayze the results and output a comprehensive and insightful report"
+            ],
+            markdown=True,
+        )
+        self.ml_task = ml_task
+        _ = self.load_data(data_source=data_source)
+
+    def load_data(self, data_source: str) -> Union[None, bool]:
+        '''Load CSV into dataframe'''
+        try:
+            self.data = pd.read_csv(data_source)
+            return True
+        except Exception as e:
+            logger.error(f"Failed to read the file from the data source with error: {e}", log_type="data_quality_assessment", console=True)
+            return False
+        
+    def detect_variable_types(self)->Tuple[Dict[str, str], Dict[str, str]]:
+        """Automatically detect continuous vs categorical variables"""
+        continuous_vars = {}
+        categorical_vars = {}
+        
+        for col in self.data.columns:
+            col_data = self.data[col].dropna()
+            
+            if len(col_data) == 0:
+                continue
+                
+            dtype = str(self.data[col].dtype)
+            nunique = col_data.nunique()
+            sample_values = col_data.head(5).tolist()
+            
+            if np.issubdtype(self.data[col].dtype, np.datetime64):
+                continue
+                
+            if dtype == 'bool' or nunique == 2 and set(col_data.unique()).issubset({0, 1, True, False}):
+                categorical_vars[col] = 'binary'
+                continue
+                
+            if dtype == 'object':
+                try:
+                    _ = pd.to_numeric(col_data)
+                    col_data = pd.to_numeric(col_data)
+                except ValueError:
+                    categorical_vars[col] = f'categorical (text, {nunique} unique values)'
+                    continue
+                    
+            if np.issubdtype(col_data.dtype, np.number):
+                if nunique <= 20:
+                    if (np.array_equal(col_data.unique(), np.arange(nunique)) or (nunique <= 10 and all(x in col_data.unique() for x in range(1, nunique+1)))):
+                        categorical_vars[col] = f'categorical (ordinal, {nunique} levels)'
+                    else:
+                        categorical_vars[col] = f'categorical (nominal, {nunique} levels)'
+                else:
+                    continuous_vars[col] = 'continuous'
+            else:
+                categorical_vars[col] = f'categorical (other, {nunique} unique values)'
+                
+        return continuous_vars, categorical_vars
+    
+    def analyze_distributions(self, verbose=False):
+        """Run distribution analysis for all continuous variables"""
+        logger.info("Starting to analyze distributions..", log_type='univariate_analysis', console=verbose)
+        continuous_vars, _ = self.detect_variable_types()
+        results = {}
+        for var in tqdm(continuous_vars):
+            try:
+                var_results = {
+                    'kde': self.kernel_density_estimation(var),
+                    'normality_tests': self.run_normality_tests(var),
+                    # 'distribution_fit': self.fit_distributions(var),
+                    # 'modality_tests': self.test_modality(var)
+                }
+                results[var] = var_results
+            except Exception as e:
+                if verbose:
+                    logger.error(f"Error analyzing variable {var}: {str(e)}", log_type="data_quality_assessment", console=verbose)
+                results[var] = {'error': str(e)}
+        return results
+    
+    def kernel_density_estimation(self, variable: str, verbose: bool = False) -> Dict[str, Any]:
+        """Perform comprehensive kernel density estimation with optimal bandwidth selection, multiple kernel types, and comparative analysis."""
+        if variable not in self.data.columns:
+            raise ValueError(f"Variable {variable} not found in dataset")
+
+        x = self.data[variable].dropna().values
+        results = {}
+
+        try:
+            results['basic_stats'] = {
+                'n': len(x),
+                'mean': np.mean(x),
+                'std': np.std(x),
+                'min': np.min(x),
+                'max': np.max(x),
+                'skewness': stats.skew(x),
+                'kurtosis': stats.kurtosis(x)
+            }
+        except Exception as e:
+            logger.error(f"Failed to compute basic stats: {e}", log_type='univariate_analysis', console=verbose)
+
+        try:
+            silverman_bandwidth = (4 * np.std(x)**5 / (3 * len(x)))**(1/5)
+            scott_bandwidth = 1.06 * np.std(x) * len(x)**(-1/5)
+        except Exception as e:
+            silverman_bandwidth = scott_bandwidth = None
+            logger.error(f"Failed to compute Silverman/Scott bandwidths: {e}", log_type='univariate_analysis', console=verbose)
+
+        try:
+            grid = GridSearchCV(
+                KernelDensity(kernel='gaussian'),
+                {'bandwidth': np.linspace(0.1, 2, 30)},
+                cv=5
+            )
+            grid.fit(x.reshape(-1, 1))
+            cv_bandwidth = grid.best_params_['bandwidth']
+        except Exception as e:
+            cv_bandwidth = None
+            logger.error(f"Grid search for bandwidth failed: {e}", log_type='univariate_analysis', console=verbose)
+
+        results['bandwidth_selection'] = {
+            'silverman': silverman_bandwidth,
+            'scott': scott_bandwidth,
+            'cross_validated': cv_bandwidth,
+            'selected_bandwidth': cv_bandwidth or silverman_bandwidth or scott_bandwidth
+        }
+        bandwidth = results['bandwidth_selection']['selected_bandwidth']
+
+        kernels = ['gaussian', 'tophat', 'epanechnikov', 'exponential', 'linear', 'cosine']
+        kde_models = {}
+        for kernel in kernels:
+            try:
+                kde = KernelDensity(kernel=kernel, bandwidth=bandwidth)
+                kde.fit(x.reshape(-1, 1))
+                kde_models[kernel] = kde
+            except Exception as e:
+                logger.error(f"Failed to fit KDE with kernel {kernel}: {e}", log_type='univariate_analysis', console=verbose)
+
+        try:
+            x_grid = np.linspace(np.min(x), np.max(x), 1000)
+            log_likelihoods = {}
+            for kernel, model in kde_models.items():
+                log_likelihoods[kernel] = model.score_samples(x_grid.reshape(-1, 1)).sum()
+
+            results['kernel_comparison'] = {
+                'log_likelihoods': log_likelihoods,
+                'best_kernel': max(log_likelihoods, key=log_likelihoods.get)
+            }
+            best_kernel = results['kernel_comparison']['best_kernel']
+            best_kde = kde_models[best_kernel]
+            density = np.exp(best_kde.score_samples(x_grid.reshape(-1, 1)))
+        except Exception as e:
+            results['kernel_comparison'] = {}
+            density = None
+            x_grid = None
+            logger.error(f"Kernel comparison failed: {e}", log_type='univariate_analysis', console=verbose)
+
+
+        dist_names = ['norm', 'lognorm', 'expon', 'gamma', 'beta']
+        dist_results = []
+        for dist_name in dist_names:
+            try:
+                params = getattr(stats, dist_name).fit(x)
+                pdf = getattr(stats, dist_name).pdf(x_grid, *params)
+                kl_div = stats.entropy((density + 1e-10), (pdf + 1e-10))
+                dist_results.append({
+                    'distribution': dist_name,
+                    'params': params,
+                    'kl_divergence': kl_div
+                })
+            except Exception as e:
+                logger.error(f"Failed to fit and evaluate {dist_name}: {e}", log_type='univariate_analysis', console=verbose)
+
+
+        try:
+            results['parametric_comparison'] = sorted(dist_results, key=lambda x: x['kl_divergence'])
+            if results['parametric_comparison']:
+                best_dist = results['parametric_comparison'][0]
+                dist_pdf = getattr(stats, best_dist['distribution']).pdf(x_grid, *best_dist['params'])
+        except Exception as e:
+            results['parametric_comparison'] = []
+            logger.error(f"Parametric comparison failed: {e}", log_type='univariate_analysis', console=verbose)
+
+
+        try:
+            bandwidths = np.linspace(bandwidth * 0.1, bandwidth * 2, 20)
+            log_liks = []
+            for bw in bandwidths:
+                kde = KernelDensity(kernel=best_kernel, bandwidth=bw)
+                kde.fit(x.reshape(-1, 1))
+                log_liks.append(kde.score(x.reshape(-1, 1)))
+            results['bandwidth_sensitivity'] = {
+                'bandwidths': bandwidths.tolist(),
+                'log_likelihoods': log_liks
+            }
+        except Exception as e:
+            logger.error(f"Bandwidth sensitivity analysis failed: {e}", log_type='univariate_analysis', console=verbose)
+
+        results['summary'] = self.summarize_kde_results(kde_results=results, variable=variable)
+
+        return results
+
+    def summarize_kde_results(self, kde_results: Dict[str, Any], variable: str) -> Dict[str, Any]:
+        """Generate a precise and insightful summary of kernel density estimation results"""
+        
+        summary = {
+            'variable': variable,
+            'distribution_characteristics': {},
+            'best_fit': {},
+            'technical_details': {}
+        }
+
+        try:
+            if 'basic_stats' in kde_results:
+                stats = kde_results['basic_stats']
+                shape = "symmetric"
+                if stats.get('skewness') is not None:
+                    if stats['skewness'] > 0.5:
+                        shape = "right-skewed"
+                    elif stats['skewness'] < -0.5:
+                        shape = "left-skewed"
+                peakedness = "mesokurtic"
+                if stats.get('kurtosis') is not None:
+                    if stats['kurtosis'] > 0.5:
+                        peakedness = "leptokurtic (heavy-tailed)"
+                    elif stats['kurtosis'] < -0.5:
+                        peakedness = "platykurtic (light-tailed)"
+                range_width = stats.get('max', 0) - stats.get('min', 0)
+                std_ranges = range_width / stats.get('std', 1) if stats.get('std') else None
+                summary['distribution_characteristics'] = {
+                    'shape': shape,
+                    'peakedness': peakedness,
+                    'central_tendency': stats.get('mean'),
+                    'dispersion': stats.get('std'),
+                    'range_in_std_units': round(std_ranges, 2) if std_ranges else None,
+                    'sample_size': stats.get('n')
+                }
+        except Exception as e:
+            if variable:
+                summary['distribution_characteristics'] = {'error': str(e)}
+
+        try:
+            if 'kernel_comparison' in kde_results and kde_results['kernel_comparison']:
+                summary['best_fit']['nonparametric'] = {
+                    'method': 'kernel_density',
+                    'best_kernel': kde_results['kernel_comparison'].get('best_kernel')
+                }
+        except Exception as e:
+            summary['best_fit']['nonparametric'] = {'error': str(e)}
+
+        try:
+            if 'parametric_comparison' in kde_results and kde_results['parametric_comparison']:
+                best_dist = kde_results['parametric_comparison'][0]
+                kl_div = best_dist.get('kl_divergence', None)
+                summary['best_fit']['parametric'] = {
+                    'distribution': best_dist.get('distribution'),
+                    'kl_divergence': round(kl_div, 4) if kl_div else None
+                }
+                if kl_div is not None:
+                    if kl_div < 0.05:
+                        fit_quality = "excellent"
+                    elif kl_div < 0.1:
+                        fit_quality = "good"
+                    elif kl_div < 0.3:
+                        fit_quality = "moderate"
+                    else:
+                        fit_quality = "poor"
+                    summary['best_fit']['parametric']['fit_quality'] = fit_quality
+        except Exception as e:
+            summary['best_fit']['parametric'] = {'error': str(e)}
+
+        try:
+            if 'bandwidth_selection' in kde_results:
+                bw = kde_results['bandwidth_selection']
+                selected = bw.get('selected_bandwidth')
+                method = next((k for k, v in bw.items() 
+                            if v == selected and k != 'selected_bandwidth'), None)
+                summary['technical_details']['bandwidth'] = {
+                    'selected': selected,
+                    'selection_method': method
+                }
+        except Exception as e:
+            summary['technical_details']['bandwidth'] = {'error': str(e)}
+
+        try:
+            if 'basic_stats' in kde_results and 'bandwidth_selection' in kde_results:
+                range_width = kde_results['basic_stats'].get('max', 0) - kde_results['basic_stats'].get('min', 0)
+                selected_bw = kde_results['bandwidth_selection'].get('selected_bandwidth', 0)
+                if selected_bw and range_width:
+                    relative_bw = selected_bw / range_width
+                    if relative_bw < 0.03:
+                        multimodality = "highly likely"
+                    elif relative_bw < 0.06:
+                        multimodality = "possible"
+                    else:
+                        multimodality = "unlikely"
+                    summary['distribution_characteristics']['multimodality'] = multimodality
+        except Exception as e:
+            summary['distribution_characteristics']['multimodality_error'] = str(e)
+
+        try:
+            recommendations = []
+            if 'basic_stats' in kde_results and kde_results['basic_stats'].get('n', 0) < 30:
+                recommendations.append("Sample size is small; interpret results with caution")
+            if ('basic_stats' in kde_results and 
+                abs(kde_results['basic_stats'].get('skewness', 0)) > 1.5 or 
+                abs(kde_results['basic_stats'].get('kurtosis', 0)) > 2):
+                recommendations.append("Distribution has extreme values; consider transformation or robust methods")
+            if ('parametric_comparison' in kde_results and kde_results['parametric_comparison'] and 
+                kde_results['parametric_comparison'][0].get('kl_divergence', 1) < 0.1):
+                best_dist = kde_results['parametric_comparison'][0]['distribution']
+                recommendations.append(f"Consider using {best_dist} distribution for parametric modeling")
+            summary['recommendations'] = recommendations
+        except Exception as e:
+            summary['recommendations'] = [f"Failed to generate recommendations: {str(e)}"]
+
+        return summary
+
+    def run_normality_tests(self, variable, verbose=False):
+        """Run battery of normality tests"""
+        try:
+            data = self.data[variable].dropna()
+            sample_size = len(data)
+            
+            if len(self.data) > 10000:
+                sample_size = 1000 
+            elif 5000 < len(self.data) <= 10000:
+                sample_size = int(0.2 * len(self.data))
+            else:
+                sample_size = len(self.data)
+            
+            if sample_size < len(data):
+                data = data.sample(n=sample_size, random_state=42)
+            
+            normality_tests = {}
+            
+            try:
+                if sample_size < 30:
+                    normality_tests['shapiro_wilk'] = self.shapiro_wilk_normality_test(data)
+                    normality_tests['dagostino_pearson'] = self.dagostino_pearson_normality_test(data)
+                elif 30 <= sample_size <= 2000:
+                    normality_tests['shapiro_wilk'] = self.shapiro_wilk_normality_test(data)
+                    normality_tests['dagostino_pearson'] = self.dagostino_pearson_normality_test(data)
+                    normality_tests['anderson_darling'] = self.anderson_darling_normality_test(data)
+                elif sample_size > 2000:
+                    normality_tests['dagostino_pearson'] = self.dagostino_pearson_normality_test(data)
+                    normality_tests['anderson_darling'] = self.anderson_darling_normality_test(data)
+                    normality_tests['kolmogorov_smirnov'] = self.kolmogorov_smirnov_normality_test(data)
+            except Exception as e:
+                logger.error(f"Error running normality tests for {variable}: {str(e)}", log_type="data_quality_assessment", console=verbose)
+                normality_tests['error'] = str(e)
+            
+            normality_tests['summary'] = self.generate_normality_test_summary(normality_tests, variable)
+
+            return normality_tests
+        
+        except Exception as e:
+            logger.error(f"Error processing variable {variable}: {str(e)}", log_type="data_quality_assessment", console=verbose)
+            return {'error': str(e)}
+
+    def shapiro_wilk_normality_test(self, data):
+        """Perform Shapiro-Wilk test for normality"""
+        try:
+            stat, p = shapiro(data)
+            return {
+                'test_statistic': stat,
+                'p_value': p,
+                'interpretation': 'Data looks normally distributed (fail to reject H0)' if p > 0.05 else 'Data does not look normally distributed (reject H0)'
+            }
+        except Exception as e:
+            return {
+                'error': f"Shapiro-Wilk test failed: {str(e)}",
+                'test_statistic': None,
+                'p_value': None
+            }
+
+    def dagostino_pearson_normality_test(self, data):
+        """Perform D'Agostino-Pearson test for normality"""
+        try:
+            stat, p = normaltest(data)
+            return {
+                'test_statistic': stat,
+                'p_value': p,
+                'interpretation': 'Data looks normally distributed (fail to reject H0)' if p > 0.05 else 'Data does not look normally distributed (reject H0)'
+            }
+        except Exception as e:
+            return {
+                'error': f"D'Agostino-Pearson test failed: {str(e)}",
+                'test_statistic': None,
+                'p_value': None
+            }
+
+    def anderson_darling_normality_test(self, data):
+        """Perform Anderson-Darling test for normality"""
+        try:
+            result = anderson(data, dist='norm')
+            return {
+                'test_statistic': result.statistic,
+                'critical_values': result.critical_values,
+                'significance_levels': result.significance_level,
+                'interpretation': 'Data looks normally distributed at 5% level' if result.statistic < result.critical_values[2] else 'Data does not look normally distributed at 5% level'
+            }
+        except Exception as e:
+            return {
+                'error': f"Anderson-Darling test failed: {str(e)}",
+                'test_statistic': None,
+                'critical_values': None
+            }
+
+    def kolmogorov_smirnov_normality_test(self, data):
+        """Perform Kolmogorov-Smirnov test for normality"""
+        try:
+            mu, std = norm.fit(data)
+            
+            stat, p = kstest(data, 'norm', args=(mu, std))
+            
+            return {
+                'test_statistic': stat,
+                'p_value': p,
+                'interpretation': 'Data looks normally distributed (fail to reject H0)' if p > 0.05 else 'Data does not look normally distributed (reject H0)'
+            }
+        except Exception as e:
+            return {
+                'error': f"Kolmogorov-Smirnov test failed: {str(e)}",
+                'test_statistic': None,
+                'p_value': None
+            }
+
+    def generate_normality_test_summary(self, normality_tests_results, variable_name):
+        """Generate a structured and insightful summary of normality test results"""
+        try:
+            if 'error' in normality_tests_results:
+                return {
+                    'variable': variable_name,
+                    'normality_assessment': 'Error in analysis',
+                    'error_message': normality_tests_results['error'],
+                    'recommendation': 'Check the data quality or transform the variable'
+                }
+        except Exception as e:
+            return {
+                'variable': variable_name,
+                'normality_assessment': 'Error in checking error flag',
+                'error_message': str(e),
+                'recommendation': 'Ensure error key structure is valid'
+            }
+
+        total_tests = 0
+        normal_tests = 0
+        tests_performed = []
+        test_details = {}
+
+        try:
+            for test_name, result in normality_tests_results.items():
+                try:
+                    if 'error' in result:
+                        test_details[test_name] = {
+                            'status': 'Failed',
+                            'reason': result['error']
+                        }
+                        continue
+
+                    total_tests += 1
+                    tests_performed.append(test_name)
+
+                    is_normal = False
+
+                    if test_name in ['shapiro_wilk', 'dagostino_pearson', 'kolmogorov_smirnov']:
+                        is_normal = result.get('p_value', 0) > 0.05
+                        test_details[test_name] = {
+                            'test_statistic': result.get('test_statistic'),
+                            'p_value': result.get('p_value'),
+                            'suggests_normality': is_normal
+                        }
+
+                    elif test_name == 'anderson_darling':
+                        is_normal = result.get('test_statistic', float('inf')) < result.get('critical_values', [0, 0, 0])[2]
+                        test_details[test_name] = {
+                            'test_statistic': result.get('test_statistic'),
+                            'critical_value_5pct': result.get('critical_values', [0, 0, 0])[2],
+                            'suggests_normality': is_normal
+                        }
+
+                    if is_normal:
+                        normal_tests += 1
+
+                except Exception as e:
+                    test_details[test_name] = {
+                        'status': 'Failed',
+                        'reason': f"Exception during processing: {str(e)}"
+                    }
+        except Exception as e:
+            return {
+                'variable': variable_name,
+                'normality_assessment': 'Error in parsing test results',
+                'error_message': str(e),
+                'recommendation': 'Ensure the test result structure is consistent'
+            }
+
+        try:
+            normality_score = normal_tests / total_tests if total_tests > 0 else 0
+
+            if normality_score >= 0.75:
+                normality_assessment = 'Normal'
+                confidence = 'High'
+                recommendation = 'Proceed with parametric tests'
+            elif normality_score >= 0.5:
+                normality_assessment = 'Likely Normal'
+                confidence = 'Moderate'
+                recommendation = 'Consider parametric tests, but verify with visual inspection'
+            elif normality_score >= 0.25:
+                normality_assessment = 'Likely Non-Normal'
+                confidence = 'Moderate'
+                recommendation = 'Consider non-parametric alternatives or data transformation'
+            else:
+                normality_assessment = 'Non-Normal'
+                confidence = 'High'
+                recommendation = 'Use non-parametric tests or transform the data'
+        except Exception as e:
+            return {
+                'variable': variable_name,
+                'normality_assessment': 'Error in calculating summary statistics',
+                'error_message': str(e),
+                'recommendation': 'Check calculations or inputs for division errors'
+            }
+
+        try:
+            summary = {
+                'variable': variable_name,
+                'normality_assessment': normality_assessment,
+                'confidence': confidence,
+                'normal_tests_ratio': f"{normal_tests}/{total_tests}",
+                'normality_score': normality_score,
+                'tests_performed': tests_performed,
+                'test_details': test_details,
+                'recommendation': recommendation
+            }
+        except Exception as e:
+            return {
+                'variable': variable_name,
+                'normality_assessment': 'Error in creating summary',
+                'error_message': str(e),
+                'recommendation': 'Inspect result structure and try again'
+            }
+
+        return summary
+
+    def fit_distributions(self, variable, verbose=False):
+        """Fit common distributions and assess goodness-of-fit"""
+        pass
+
+    def test_modality(self, variable, verbose=False):
+            """ Test for unimodal vs multimodal distributions using Hartigan's Dip Test"""
+            try:
+                x = np.asarray(variable).flatten()
+                
+                x = x[~np.isnan(x)]
+                
+                if len(x) <= 3:
+                    if verbose:
+                        logger.info("Warning: Too few data points for reliable modality testing.", log_type='univariate_analysis', console=verbose)
+                    return {
+                        'dip_statistic': None,
+                        'p_value': None,
+                        'conclusion': "Not enough data points for reliable testing (minimum 4 required)",
+                        'is_multimodal': None,
+                        'sample_size': len(x)
+                    }
+                
+                dip, p_value = self.hartigan_dip_test(x)
+                
+                alpha = 0.05
+                if p_value < alpha:
+                    conclusion = "Likely multimodal distribution (reject unimodality)"
+                    is_multimodal = True
+                else:
+                    conclusion = "Likely unimodal distribution (fail to reject unimodality)"
+                    is_multimodal = False
+                    
+                results = {
+                    'dip_statistic': dip,
+                    'p_value': p_value,
+                    'conclusion': conclusion,
+                    'is_multimodal': is_multimodal,
+                    'sample_size': len(x)
+                } 
+
+                results['summary'] = self.summarize_modality_test(results)
+
+                return results
+            
+            except Exception as e:
+                logger.error(f"Failed to run modality test with error: {e}", log_type="univariate_analysis", console=verbose)
+                return {"error": str(e)}
+    
+    def hartigan_dip_test(self, x: np.ndarray) -> Tuple[float, float]:
+        """Implement Hartigan's dip test for unimodality """
+
+        x = np.sort(x)
+        n = len(x)
+        
+        ecdf = np.arange(1, n + 1) / n
+        
+        dip = self.calculate_dip_statistic(x, ecdf)
+        
+        p_value = self.calculate_dip_pvalue(dip, n)
+        
+        return dip, p_value
+    
+    def calculate_dip_statistic(self, x: np.ndarray, ecdf: np.ndarray) -> float:
+        """Calculate the Hartigan's dip statistic """
+        
+        n = len(x)
+        gcm = self.compute_gcm(x, ecdf)
+        
+        lcm = self.compute_lcm(x, ecdf)
+        
+        diffs = np.maximum(
+            np.abs(ecdf - gcm),
+            np.abs(ecdf - lcm)
+        )
+        
+        dip = np.max(diffs)
+        
+        return dip
+    
+    def compute_gcm(self, x: np.ndarray, ecdf: np.ndarray) -> np.ndarray:
+        """Compute the greatest convex minorant (GCM) of the empirical CDF"""
+        n = len(x)
+        gcm = np.zeros(n)
+        
+        gcm[0] = ecdf[0]
+        
+        for i in range(1, n):
+            slopes = (ecdf[i] - gcm[:i]) / (x[i] - x[:i])
+            max_slope_idx = np.argmax(slopes)
+            gcm[i] = gcm[max_slope_idx] + slopes[max_slope_idx] * (x[i] - x[max_slope_idx])
+            
+            gcm[i] = min(gcm[i], ecdf[i])
+        
+        return gcm
+    
+    def compute_lcm(self, x: np.ndarray, ecdf: np.ndarray) -> np.ndarray:
+        """Compute the least concave majorant (LCM) of the empirical CDF"""
+        n = len(x)
+        lcm = np.zeros(n)
+        
+        lcm[-1] = ecdf[-1]
+        
+        for i in range(n-2, -1, -1):
+            slopes = (lcm[i+1:] - ecdf[i]) / (x[i+1:] - x[i])
+            min_slope_idx = np.argmin(slopes) + i + 1
+            lcm[i] = ecdf[i] + slopes[min_slope_idx - i - 1] * (x[min_slope_idx] - x[i])
+            
+            lcm[i] = max(lcm[i], ecdf[i])
+        
+        return lcm
+    
+    def calculate_dip_pvalue(self, dip: float, n: int) -> float:
+        """Calculate the p-value for the dip statistic"""
+        adjusted_dip = dip * (np.sqrt(n) + 0.12 + 0.11 / np.sqrt(n))
+        
+        if adjusted_dip < 0.01:
+            p_value = 1.0
+        elif adjusted_dip < 0.7:
+            p_value = np.exp(-4.0 * adjusted_dip * adjusted_dip)
+        elif adjusted_dip < 0.9:
+            p_value = 0.15
+        elif adjusted_dip < 1.0:
+            p_value = 0.10
+        elif adjusted_dip < 1.1:
+            p_value = 0.05
+        elif adjusted_dip < 1.2:
+            p_value = 0.025
+        elif adjusted_dip < 1.3:
+            p_value = 0.01
+        elif adjusted_dip < 1.5:
+            p_value = 0.005
+        else:
+            p_value = 0.001
+            
+        return p_value
+
+    def summarize_modality_test(self, test_results: Dict) -> Dict:
+        """Generate a precise and insightful summary of modality test results"""
+        
+        if test_results.get('dip_statistic') is None:
+            return {
+                'summary': "Insufficient data for modality testing.",
+                'recommendation': "Collect more data points (minimum 4 required).",
+                'reliability': "Not applicable"
+            }
+        
+        dip = test_results['dip_statistic']
+        p_value = test_results['p_value']
+        n = test_results['sample_size']
+        is_multimodal = test_results['is_multimodal']
+        
+        confidence_level = 1 - p_value
+        
+        if p_value < 0.001:
+            strength = "Very strong"
+            reliability = "High"
+        elif p_value < 0.01:
+            strength = "Strong"
+            reliability = "High"
+        elif p_value < 0.05:
+            strength = "Moderate"
+            reliability = "Moderate"
+        elif p_value < 0.1:
+            strength = "Weak"
+            reliability = "Low"
+        else:
+            strength = "Very weak"
+            reliability = "Low"
+            
+        if n < 20:
+            reliability = "Low (small sample size)"
+        elif n < 50:
+            reliability = reliability + " (moderate sample size)"
+        else:
+            reliability = reliability + " (large sample size)"
+            
+        if is_multimodal:
+            conclusion = f"{strength} evidence of multimodality (dip={dip:.6f}, p={p_value:.6f})"
+            recommendation = "Consider mixture modeling or clustering approaches."
+        else:
+            conclusion = f"{strength} evidence for unimodality (dip={dip:.6f}, p={p_value:.6f})"
+            recommendation = "Parametric distributional analyses may be appropriate."
+            
+        return {
+            'summary': conclusion,
+            'dip_statistic_interpretation': f"Dip statistic of {dip:.6f} indicates " + 
+                                        ("significant" if is_multimodal else "non-significant") + 
+                                        " deviation from unimodality.",
+            'confidence_level': f"{confidence_level:.1%}",
+            'evidence_strength': strength,
+            'reliability': reliability,
+            'recommendation': recommendation,
+            'sample_size_adequacy': "Adequate" if n >= 50 else "Limited" if n >= 20 else "Inadequate",
+            'technical_details': {
+                'dip_statistic': dip,
+                'p_value': p_value,
+                'sample_size': n,
+                'critical_alpha': 0.05,
+                'null_hypothesis': "The distribution is unimodal",
+                'alternative_hypothesis': "The distribution is multimodal"
+            }
+        }
+
+    def calculate_shape_metrics(self, continuous_cols: List[str] = None, verbose=False):
+        """Calculate shape characteristics for all continuous variables"""
+        logger.info("Starting to calculate shape metrics..", log_type='univariate_analysis', console=verbose)
+        results = {
+                "skewness": {},
+                "kurtosis": {},
+                "l_moments": {},
+                "entropy": {},
+                "tail_weight": {},
+                "zero_variance": {},
+                "summary": {}
+            }
+            
+        if continuous_cols is None:
+            try:
+                continuous_cols = df.select_dtypes(include=['number']).columns.tolist()
+            except Exception as e:
+                logger.error(f"Error detecting continuous columns: {str(e)}", log_type="data_quality_assessment", console=verbose)
+                continuous_cols = []
+        
+        if not continuous_cols:
+            return {"error": "No continuous columns found or provided"}
+        
+        for col in continuous_cols:
+            try:
+                values = self.data[col].dropna().values
+                
+                if len(values) == 0:
+                    results["summary"][col] = "All values are NA"
+                    continue
+                    
+                col_results = {}
+                
+                try:
+                    skewness = stats.skew(values)
+                    skewness_interpretation = self.interpret_skewness(skewness)
+                    results["skewness"][col] = {
+                        "value": float(skewness),
+                        "interpretation": skewness_interpretation
+                    }
+                    col_results["skewness"] = skewness_interpretation
+                except Exception as e:
+                    results["skewness"][col] = {"error": str(e)}
+                    col_results["skewness"] = "Error calculating"
+                
+                try:
+                    kurtosis = stats.kurtosis(values)
+                    kurtosis_interpretation = self.interpret_kurtosis(kurtosis)
+                    results["kurtosis"][col] = {
+                        "value": float(kurtosis),
+                        "interpretation": kurtosis_interpretation
+                    }
+                    col_results["kurtosis"] = kurtosis_interpretation
+                except Exception as e:
+                    results["kurtosis"][col] = {"error": str(e)}
+                    col_results["kurtosis"] = "Error calculating"
+                
+                try:
+                    l_moments = self.calculate_l_moments(values)
+                    results["l_moments"][col] = l_moments
+                    col_results["l_moments"] = f"L-CV: {l_moments['l_cv']:.3f}, L-skewness: {l_moments['l_skewness']:.3f}, L-kurtosis: {l_moments['l_kurtosis']:.3f}"
+                except Exception as e:
+                    results["l_moments"][col] = {"error": str(e)}
+                    col_results["l_moments"] = "Error calculating"
+                
+                try:
+                    entropy_values = self.calculate_entropy(values)
+                    results["entropy"][col] = entropy_values
+                    col_results["entropy"] = f"Shannon: {entropy_values['shannon']:.3f}, Differential: {entropy_values['differential']:.3f}"
+                except Exception as e:
+                    results["entropy"][col] = {"error": str(e)}
+                    col_results["entropy"] = "Error calculating"
+                
+                try:
+                    tail_weights = self.assess_tail_weight(values)
+                    results["tail_weight"][col] = tail_weights
+                    col_results["tail_weight"] = f"Left: {tail_weights['left_tail']:.3f}, Right: {tail_weights['right_tail']:.3f}"
+                except Exception as e:
+                    results["tail_weight"][col] = {"error": str(e)}
+                    col_results["tail_weight"] = "Error calculating"
+                
+                try:
+                    zero_var_results = self.test_zero_variance(values)
+                    results["zero_variance"][col] = zero_var_results
+                    if zero_var_results["is_near_zero_variance"]:
+                        col_results["zero_variance"] = "Near-zero variance detected"
+                    else:
+                        col_results["zero_variance"] = "Variable has sufficient variance"
+                except Exception as e:
+                    results["zero_variance"][col] = {"error": str(e)}
+                    col_results["zero_variance"] = "Error calculating"
+                
+                results["summary"][col] = self.generate_column_summary(col, col_results)
+                
+            except Exception as e:
+                results["summary"][col] = f"Error processing column {col}: {str(e)}"
+                if verbose:
+                    print(f"Error processing column {col}: {str(e)}")
+        
+        results["overall_summary"] = self.generate_overall_summary(results)
+        
+        return results
+    
+    def interpret_skewness(self, skewness: float) -> str:
+        """Interpret the skewness value."""
+        if abs(skewness) < 0.5:
+            return "Approximately symmetric"
+        elif skewness < -1:
+            return "Highly negatively skewed"
+        elif skewness < -0.5:
+            return "Moderately negatively skewed"
+        elif skewness > 1:
+            return "Highly positively skewed"
+        else:  # skewness > 0.5
+            return "Moderately positively skewed"
+        
+    def interpret_kurtosis(self, kurtosis: float) -> str:
+        """Interpret the kurtosis value."""
+        if abs(kurtosis) < 0.5:
+            return "Approximately mesokurtic (normal-like tails)"
+        elif kurtosis < -0.5:
+            return "Platykurtic (thinner tails than normal)"
+        else:  # kurtosis > 0.5
+            return "Leptokurtic (heavier tails than normal)"
+        
+    def calculate_l_moments(self, values: np.ndarray) -> Dict:
+        """Calculate L-moments for a sample."""
+        try:
+            lmoms = lm.lmom_ratios(values, nmom=5)
+            return {
+                "l1": float(lmoms[0]),  # L-mean
+                "l2": float(lmoms[1]),  # L-scale
+                "l_cv": float(lmoms[1] / lmoms[0]) if lmoms[0] != 0 else float('nan'),  # L-coefficient of variation
+                "l_skewness": float(lmoms[2]),  # L-skewness
+                "l_kurtosis": float(lmoms[3]),  # L-kurtosis
+                "tau5": float(lmoms[4]) if len(lmoms) > 4 else float('nan')  # 5th L-moment ratio
+            }
+        except Exception:
+            try:
+                sorted_data = np.sort(values)
+                n = len(sorted_data)
+                
+                l1 = np.mean(sorted_data)
+                
+                cumsum = np.cumsum(sorted_data)
+                indices = np.arange(n)
+                l2 = np.mean((indices / (n - 1)) * sorted_data - cumsum / (n - 1))
+                
+                return {
+                    "l1": float(l1),
+                    "l2": float(l2),
+                    "l_cv": float(l2 / l1) if l1 != 0 else float('nan'),
+                    "l_skewness": float('nan'),
+                    "l_kurtosis": float('nan'),
+                    "tau5": float('nan')
+                }
+            except Exception as e:
+                logger.error(f"Error calculating L-moments: {str(e)}", log_type="data_quality_assessment", console=True)
+    
+    def calculate_entropy(self, values: np.ndarray) -> Dict:
+        """Calculate entropy measures for a sample."""
+        try:
+            hist, bin_edges = np.histogram(values, bins='auto', density=True)
+            hist = hist[hist > 0]
+            shannon_entropy = -np.sum(hist * np.log(hist)) * (bin_edges[1] - bin_edges[0])
+            
+            n = len(values)
+            std = np.std(values)
+            diff_entropy = 0.5 * np.log(2 * np.pi * np.e * std**2) if std > 0 else float('-inf')
+            
+            return {
+                "shannon": float(shannon_entropy),
+                "differential": float(diff_entropy),
+                "normalized_shannon": float(shannon_entropy / np.log(len(hist))) if len(hist) > 1 else 0.0
+            }
+        except Exception as e:
+            logger.error(f"Error calculating entropy: {str(e)}", log_type="data_quality_assessment", console=True)
+
+    def assess_tail_weight(self, values: np.ndarray) -> Dict:
+        """Assess tail weight of a distribution."""
+        try:
+            q = np.quantile(values, [0.01, 0.05, 0.25, 0.5, 0.75, 0.95, 0.99])
+            
+            iqr = q[4] - q[2]  # 75th - 25th
+            
+            left_tail = (q[2] - q[0]) / iqr if iqr > 0 else float('inf') 
+            right_tail = (q[6] - q[4]) / iqr if iqr > 0 else float('inf') 
+            
+            left_sorted = np.sort(values - np.min(values) + 1e-10) 
+            left_sorted = left_sorted[:int(0.2 * len(left_sorted))]  
+            left_tail_index = 1 / np.mean(np.log(left_sorted[-1] / left_sorted)) if len(left_sorted) > 1 else float('nan')
+            
+            right_sorted = np.sort(-values + np.max(values) + 1e-10)  
+            right_sorted = right_sorted[:int(0.2 * len(right_sorted))] 
+            right_tail_index = 1 / np.mean(np.log(right_sorted[-1] / right_sorted)) if len(right_sorted) > 1 else float('nan')
+            
+            interpretation = "Symmetric tails"
+            if abs(left_tail - right_tail) > 0.5:
+                if left_tail > right_tail:
+                    interpretation = "Heavier left tail"
+                else:
+                    interpretation = "Heavier right tail"
+            
+            return {
+                "left_tail": float(left_tail),
+                "right_tail": float(right_tail),
+                "left_tail_index": float(left_tail_index),
+                "right_tail_index": float(right_tail_index),
+                "interpretation": interpretation
+            }
+        except Exception as e:
+            logger.error(f"Error assessing tail weight: {str(e)}", log_type="data_quality_assessment", console=True)
+
+    def test_zero_variance(self, values: np.ndarray) -> Dict:
+        """Test for zero or near-zero variance."""
+        try:
+            n = len(values)
+            variance = np.var(values)
+            
+            n_unique = len(np.unique(values))
+            unique_ratio = n_unique / n if n > 0 else 0
+            
+            if n_unique > 1:
+                counts = np.bincount(np.digitize(values, np.unique(values)))
+                sorted_counts = np.sort(counts[counts > 0])
+                freq_ratio = sorted_counts[-1] / sorted_counts[-2] if sorted_counts[-2] > 0 else float('inf')
+            else:
+                freq_ratio = float('inf')
+            
+            is_zero_variance = variance < 1e-10
+            is_near_zero_variance = (is_zero_variance or 
+                                    unique_ratio < 0.1 or  
+                                    freq_ratio > 20)       
+            
+            mean = np.mean(values)
+            cv = np.sqrt(variance) / mean if mean != 0 else float('inf')
+            
+            return {
+                "variance": float(variance),
+                "unique_ratio": float(unique_ratio),
+                "n_unique": int(n_unique),
+                "freq_ratio": float(freq_ratio),
+                "cv": float(cv),
+                "is_zero_variance": bool(is_zero_variance),
+                "is_near_zero_variance": bool(is_near_zero_variance)
+            }
+        except Exception as e:
+            logger.error(f"Error testing zero variance: {str(e)}", log_type="data_quality_assessment", console=True)
+
+    def generate_column_summary(self, col_name: str, col_results: Dict) -> str:
+        """Generate a summary string for a single column."""
+        summary = f"Column '{col_name}': "
+        
+        points = []
+        
+        if "skewness" in col_results:
+            points.append(f"distribution is {col_results['skewness'].lower()}")
+        
+        if "kurtosis" in col_results:
+            points.append(f"{col_results['kurtosis'].lower()}")
+        
+        if "tail_weight" in col_results and "tail_weight" in col_results:
+            if "Heavier" in col_results["tail_weight"]:
+                points.append(col_results["tail_weight"])
+        
+        if "zero_variance" in col_results and "Near-zero variance" in col_results["zero_variance"]:
+            points.append("WARNING: near-zero variance detected, may not be useful for modeling")
+        
+        if points:
+            summary += ", ".join(points)
+        else:
+            summary += "insufficient data to calculate shape metrics"
+        
+        return summary
+
+    def generate_overall_summary(self, results: Dict) -> str:
+        """Generate an overall summary of shape metrics for all columns."""
+        summary_lines = []
+        
+        skewed_cols = []
+        heavy_tailed_cols = []
+        near_zero_variance_cols = []
+        
+        for col, summary in results["summary"].items():
+            if "highly" in summary.lower() and "skew" in summary.lower():
+                skewed_cols.append(col)
+            
+            if "leptokurtic" in summary.lower() or "heavier tail" in summary.lower():
+                heavy_tailed_cols.append(col)
+                
+            if "zero variance" in summary.lower():
+                near_zero_variance_cols.append(col)
+        
+        summary_lines.append(f"Analyzed {len(results['summary'])} continuous variables.")
+        
+        if skewed_cols:
+            summary_lines.append(f"Found {len(skewed_cols)} highly skewed variables: {', '.join(skewed_cols[:5])}" + 
+                            (f" and {len(skewed_cols) - 5} more" if len(skewed_cols) > 5 else ""))
+        
+        if heavy_tailed_cols:
+            summary_lines.append(f"Found {len(heavy_tailed_cols)} variables with heavy tails: {', '.join(heavy_tailed_cols[:5])}" + 
+                            (f" and {len(heavy_tailed_cols) - 5} more" if len(heavy_tailed_cols) > 5 else ""))
+        
+        if near_zero_variance_cols:
+            summary_lines.append(f"WARNING: Found {len(near_zero_variance_cols)} variables with near-zero variance which may not be useful for modeling: {', '.join(near_zero_variance_cols[:5])}" + 
+                            (f" and {len(near_zero_variance_cols) - 5} more" if len(near_zero_variance_cols) > 5 else ""))
+        
+        if skewed_cols or heavy_tailed_cols:
+            summary_lines.append("Recommendation: Consider applying transformations (log, Box-Cox, etc.) to heavily skewed or heavy-tailed variables.")
+        
+        if near_zero_variance_cols:
+            summary_lines.append("Recommendation: Consider removing or carefully reviewing near-zero variance variables.")
+        
+        return "\n".join(summary_lines)
+
+    def detect_outliers(self, verbose=False):
+        """Run multiple outlier detection methods"""
+        pass
+    
+    def analyze_categoricals(
+            self,
+            data: pd.DataFrame, 
+            categorical_columns: Optional[List[str]] = None, 
+            alpha: float = 0.05, 
+            rare_threshold: float = 0.05, 
+            verbose: bool = False) -> Dict[str, Any]:
+        '''Perform comprehensive analysis on categorical variables in a dataset.'''
+        results = {
+                "frequency_tables": {},
+                "prevalence_rates": {},
+                "rare_categories": {},
+                "entropy": {},
+                "gini_impurity": {},
+                "chi_square_results": {},
+                "simpson_diversity": {},
+                "shannon_diversity": {},
+                "cardinality": {},
+                "errors": [],
+                "summary": ""
+        }
+    
+        try:
+            if categorical_columns is None:
+                categorical_columns = list(data.select_dtypes(include=['object', 'category']).columns)
+                
+                for col in data.select_dtypes(include=['int64', 'float64']).columns:
+                    if data[col].nunique() <= 30 and data[col].nunique() / len(data) < 0.05:
+                        categorical_columns.append(col)
+        except Exception as e:
+            error_msg = f"Error identifying categorical columns: {str(e)}"
+            logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+            results["errors"].append(error_msg)
+            categorical_columns = []
+        
+        if not categorical_columns:
+            results["summary"] = "No categorical columns identified for analysis."
+            return results
+        
+        for col in categorical_columns:
+            try:
+                if col not in data.columns:
+                    continue
+                    
+                missing_rate = data[col].isna().mean()
+                if missing_rate > 0.8:
+                    results["errors"].append(f"Column {col} has {missing_rate:.2%} missing values - skipping analysis")
+                    continue
+                    
+                column_data = data[col].dropna()
+                
+                if len(column_data) == 0:
+                    results["errors"].append(f"Column {col} has no valid values after removing NA - skipping analysis")
+                    continue
+                    
+                try:
+                    freq_table = column_data.value_counts(dropna=False).reset_index()
+                    freq_table.columns = ['value', 'frequency']
+                    freq_table['percentage'] = (freq_table['frequency'] / len(column_data)) * 100
+                    # results["frequency_tables"][col] = freq_table.to_dict('records')
+                except Exception as e:
+                    error_msg = f"Error calculating frequency table for {col}: {str(e)}"
+                    logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+                    results["errors"].append(error_msg)
+                
+                try:
+                    prevalence_rates = {}
+                    value_counts = column_data.value_counts(normalize=True)
+                    for value, rate in value_counts.items():
+                        prevalence_rates[str(value)] = float(rate)
+                    # results["prevalence_rates"][col] = prevalence_rates
+                except Exception as e:
+                    error_msg = f"Error calculating prevalence rates for {col}: {str(e)}"
+                    logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+                    results["errors"].append(error_msg)
+                
+                try:
+                    rare_categories = {}
+                    for value, rate in value_counts.items():
+                        if rate < rare_threshold:
+                            rare_categories[str(value)] = float(rate)
+                    results["rare_categories"][col] = rare_categories
+                except Exception as e:
+                    error_msg = f"Error detecting rare categories for {col}: {str(e)}"
+                    logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+                    results["errors"].append(error_msg)
+                
+                try:
+                    entropy_value = 0
+                    for prob in value_counts:
+                        if prob > 0:
+                            entropy_value -= prob * math.log2(prob)
+                    results["entropy"][col] = float(entropy_value)
+                except Exception as e:
+                    error_msg = f"Error calculating entropy for {col}: {str(e)}"
+                    logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+                    results["errors"].append(error_msg)
+                
+                try:
+                    gini_impurity = 1 - sum(prob ** 2 for prob in value_counts)
+                    results["gini_impurity"][col] = float(gini_impurity)
+                except Exception as e:
+                    error_msg = f"Error calculating Gini impurity for {col}: {str(e)}"
+                    logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+                    results["errors"].append(error_msg)
+                
+                try:
+
+                    observed = column_data.value_counts().values
+                    n_categories = len(observed)
+                    expected = np.ones(n_categories) * len(column_data) / n_categories
+                    
+                    if n_categories >= 2:
+                        chi2_stat, p_value = stats.chisquare(observed, expected)
+                        results["chi_square_results"][col] = {
+                            "chi2_statistic": float(chi2_stat),
+                            "p_value": float(p_value),
+                            "reject_null_hypothesis": p_value < alpha,
+                            "interpretation": "Distribution is not uniform" if p_value < alpha else "Distribution may be uniform"
+                        }
+                    else:
+                        results["chi_square_results"][col] = {
+                            "chi2_statistic": None,
+                            "p_value": None,
+                            "reject_null_hypothesis": None,
+                            "interpretation": "Not enough categories for chi-square test"
+                        }
+                except Exception as e:
+                    error_msg = f"Error performing chi-square test for {col}: {str(e)}"
+                    logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+                    results["errors"].append(error_msg)
+                
+                try:
+                    simpson_index = sum(prob ** 2 for prob in value_counts)
+                    simpson_diversity = 1 - simpson_index
+                    results["simpson_diversity"][col] = float(simpson_diversity)
+                except Exception as e:
+                    error_msg = f"Error calculating Simpson's diversity index for {col}: {str(e)}"
+                    logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+                    results["errors"].append(error_msg)
+                
+                try:
+                    shannon_diversity = -sum(prob * np.log(prob) for prob in value_counts if prob > 0)
+                    results["shannon_diversity"][col] = float(shannon_diversity)
+                except Exception as e:
+                    error_msg = f"Error calculating Shannon's diversity index for {col}: {str(e)}"
+                    logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+                    results["errors"].append(error_msg)
+                
+                try:
+                    cardinality = len(value_counts)
+                    results["cardinality"][col] = {
+                        "unique_values": int(cardinality),
+                        "total_records": int(len(column_data)),
+                        "ratio": float(cardinality / len(column_data))
+                    }
+                except Exception as e:
+                    error_msg = f"Error performing cardinality analysis for {col}: {str(e)}"
+                    logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+                    results["errors"].append(error_msg)
+                    
+            except Exception as e:
+                error_msg = f"Error analyzing column {col}: {str(e)}"
+                logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+                results["errors"].append(error_msg)
+        
+        try:
+            summary_lines = []
+            summary_lines.append(f"Categorical Analysis Summary for {len(categorical_columns)} columns:")
+            
+            if results["cardinality"]:
+                high_cardinality = [col for col, data in results["cardinality"].items() 
+                                if data["ratio"] > 0.5 and data["unique_values"] > 10]
+                if high_cardinality:
+                    summary_lines.append(f"- High cardinality columns ({len(high_cardinality)}): {', '.join(high_cardinality)}")
+            
+            if results["rare_categories"]:
+                cols_with_rare = [col for col, rare_cats in results["rare_categories"].items() if rare_cats]
+                if cols_with_rare:
+                    summary_lines.append(f"- Columns with rare categories ({len(cols_with_rare)}): {', '.join(cols_with_rare)}")
+            
+            if results["shannon_diversity"]:
+                high_diversity = [col for col, value in results["shannon_diversity"].items() if value > 2.0]
+                low_diversity = [col for col, value in results["shannon_diversity"].items() if value < 0.5]
+                if high_diversity:
+                    summary_lines.append(f"- High diversity columns ({len(high_diversity)}): {', '.join(high_diversity)}")
+                if low_diversity:
+                    summary_lines.append(f"- Low diversity columns ({len(low_diversity)}): {', '.join(low_diversity)}")
+            
+            if results["chi_square_results"]:
+                non_uniform = [col for col, result in results["chi_square_results"].items() 
+                            if result.get("reject_null_hypothesis") is True]
+                if non_uniform:
+                    summary_lines.append(f"- Columns with non-uniform distributions ({len(non_uniform)}): {', '.join(non_uniform)}")
+            
+            if results["errors"]:
+                summary_lines.append(f"- Analysis encountered {len(results['errors'])} errors during processing")
+            
+            results["summary"] = "\n".join(summary_lines)
+        except Exception as e:
+            error_msg = f"Error generating summary: {str(e)}"
+            logger.error(error_msg, log_type='univariate_analaysis', console=verbose)
+            results["errors"].append(error_msg)
+            results["summary"] = "Error generating summary."
+        
+        return results  
+
+    def analyze_all_categoricals(self, verbose=False):
+        """Analyze all categorical variables in the datase"""
+        logger.info("Starting to analyze all categories..", log_type='univariate_analysis', console=verbose)
+        try:
+            if self.data is None:
+                return {"error": "No data loaded", "summary": "Error: No data loaded"}
+            
+            categorical_columns = list(self.data.select_dtypes(include=['object', 'category']).columns)
+            
+            for col in self.data.select_dtypes(include=['int64', 'float64']).columns:
+                if self.data[col].nunique() <= 30 and self.data[col].nunique() / len(self.data) < 0.05:
+                    categorical_columns.append(col)
+            
+            results = self.analyze_categoricals(
+                data=self.data,
+                categorical_columns=categorical_columns,
+                verbose=verbose
+            )
+            
+            return results
+        
+        except Exception as e:
+            error_msg = f"Categorical analysis failed with error: {str(e)}"
+            logger.error(error_msg, log_type="univariate_analysis", console=True)
+            return {"error": error_msg, "summary": f"Error in categorical analysis: {str(e)}"}
+
+    def generate_report_from_agent(self, input)->str:
+        '''Transform the json output to a user-readable report'''
+        try:
+            input = f"ML Task: {self.ml_task}\n{input}"
+            response: RunResponse = self.writer.run(input, stream=False)
+            return response.content
+        except Exception as e:
+            return f"Failed to generate report with error: {e}"
+
+    def convert_numpy_types(self, obj):
+        if isinstance(obj, dict):
+            return {k: self.convert_numpy_types(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [self.convert_numpy_types(item) for item in obj]
+        elif isinstance(obj, np.integer):
+            return int(obj)
+        elif isinstance(obj, np.floating):
+            val = float(obj)
+            if np.isnan(val) or np.isinf(val):
+                return None  
+            return val
+        elif isinstance(obj, float):
+            if math.isnan(obj) or math.isinf(obj):
+                return None
+            return obj
+        elif isinstance(obj, np.bool_):
+            return bool(obj)
+        elif isinstance(obj, np.ndarray):
+            return self.convert_numpy_types(obj.tolist())
+        else:
+            return obj
+
+
+    def run(self, verbose=False) -> Dict[str, dict]:
+        '''Trigger point of the entire pipeline'''
+        distrbution_analysis_results = self.analyze_distributions(verbose=verbose)
+        continuous_vars, _ = self.detect_variable_types()
+        shape_metrics_analysis_results = self.calculate_shape_metrics(continuous_cols=continuous_vars, verbose=verbose)
+        categorical_analysis_results = self.analyze_all_categoricals(verbose=verbose)
+
+        kde_results = {}
+        normality_results = {}
+        kde_summary = {}
+        normality_summary = {}
+
+        for column in distrbution_analysis_results:
+            try:
+                kde_results[column] = distrbution_analysis_results[column]['kde']
+                normality_results[column] = distrbution_analysis_results[column]['normality_tests']
+                kde_summary[column] = distrbution_analysis_results[column]['kde']['summary']
+                normality_summary[column] = distrbution_analysis_results[column]['normality_tests']['summary']
+            except:
+                continue
+
+        logger.info("Generating final reports....", log_type='univariate_analysis', console=verbose)
+        
+        final_result = {
+            "kde_analysis": {
+                'dict': self.convert_numpy_types(kde_results),
+                'report': ""
+            },
+            "normality_analysis": {
+                'dict': self.convert_numpy_types(normality_results),
+                'report': ""
+            },
+            "shape_metrics_analysis": {
+                "dict": self.convert_numpy_types(shape_metrics_analysis_results),
+                "report": ""
+            },
+            "categorical_analysis": {
+                # "dict": categorical_analysis_results,
+                "report": ""
+            }
+        }
+
+        try:
+            kde_results_str = json.dumps(kde_summary, indent=2, default=str, allow_nan=True)
+            final_result['kde_analysis']['report'] = self.generate_report_from_agent(kde_results_str)
+        except:
+            logger.error("Failed to generate report for kde....", log_type='univariate_analysis', console=verbose)
+            pass
+
+        try:
+            normality_results_str = json.dumps(normality_summary, indent=2, default=str, allow_nan=True)
+            final_result['normality_analysis']['report'] = self.generate_report_from_agent(normality_results_str)
+        except:
+            logger.error("Failed to generate report for normality....", log_type='univariate_analysis', console=verbose)
+            pass
+
+        try:
+            shape_metrics_analysis_results_str = json.dumps(shape_metrics_analysis_results, indent=2, default=str, allow_nan=True)
+            final_result['shape_metrics_analysis']['report'] = self.generate_report_from_agent(shape_metrics_analysis_results_str)
+        except:
+            logger.error("Failed to generate report for shape metrics....", log_type='univariate_analysis', console=verbose)
+            pass
+
+        try:
+            categorical_analysis_results_str = json.dumps(categorical_analysis_results, indent=2, default=str, allow_nan=True)
+            final_result['categorical_analysis']['report'] = self.generate_report_from_agent(categorical_analysis_results_str)
+        except:
+            logger.error("Failed to generate report for categorical analysis....", log_type='univariate_analysis', console=verbose)
+            pass
+
+        return final_result
+
+

src/app/pipelines/task_analysis/__init__.py

@@ -0,0 +1,2 @@
+from .ml_analysis_workflow import MLAnalysisWorkflow
+from .ml_implementation_planner_workflow import MLImplementationPlannerWorkflow

src/app/pipelines/task_analysis/ml_analysis_workflow.py

@@ -0,0 +1,202 @@
+from agno.models.openai import OpenAIChat # type: ignore
+import os
+from dotenv import load_dotenv
+from src.core.utils import logger
+from typing import Optional
+from .model import RequirementsAnalysis, TechnicalResearch, ModelResponseStatus
+from agno.agent import Agent, RunResponse # type: ignore
+from agno.tools.duckduckgo import DuckDuckGoTools # type: ignore
+from typing import Iterator, List
+import json
+
+load_dotenv()
+
+class MLAnalysisWorkflow:
+    def __init__(self, user_prompt: str):
+        self.user_prompt = user_prompt
+        self.llm = OpenAIChat(id="gpt-4o", api_key=os.getenv('OPENAI_API_KEY'))
+        
+    def analyze_requirements(self, verbose=False) -> Optional[RequirementsAnalysis]:
+        """Stream requirements analysis"""
+        
+        logger.info("Analyzing requirements...", log_type="pipeline: task_analysis", console=verbose)
+        
+        prompt = f"Analyze this business problem and provide initial technical specifications: {self.user_prompt}"
+        
+        self.requirements_analyst: Agent = Agent(
+            name="ML Requirements Analyst",
+            model=self.llm,
+            description="Expert ML Solutions Architect specialized in analyzing business requirements",
+            instructions=[
+                "Analyze business problems and translate them into technical ML specifications.",
+                "1. Understand the core business problem and objectives",
+                "2. Identify the type of ML task required",
+                "3. Determine data requirements and constraints",
+                "4. List unclear points that need clarification",
+                "5. Specify areas that need technical research",
+                "Be precise in identifying what information is missing or needs validation."
+            ],
+            response_model=RequirementsAnalysis,
+            structured_outputs=True,
+            reasoning=True,
+        )
+
+        analyse_stream = self.requirements_analyst.run(prompt, stream=False)
+        return analyse_stream.content
+
+    def write_requirements_post(self, requirements_results: RequirementsAnalysis, verbose=False) -> Iterator[RunResponse]:
+        """
+        Write a blog post on a topic.
+        :param requirements_results: requirements_analyst response
+        :return: iterator for the workflow response
+        """
+        logger.info("Writing requirements analysis...", log_type="pipeline: task_analysis", console=verbose)
+       
+        writer_input = {"model_response": requirements_results.model_response.model_dump(),
+                        "unclear_points": requirements_results.unclear_points,
+                        "search_queries": requirements_results.search_queries,
+                        "business_understanding": requirements_results.business_understanding
+                        }
+        
+        model_response = self.writer.run(json.dumps(writer_input, indent=4), stream=True)
+        
+        return model_response
+
+    def write_research_post(self, research_results: TechnicalResearch, verbose=False) -> Iterator[RunResponse]:
+        """
+        Write a blog post on a topic.
+        :param research_results: research content
+        :return: iterator for the workflow response
+        """
+        logger.info("Writing research findings...", log_type="pipeline: task_analysis", console=verbose)
+
+        writer_input = {"research_findings": research_results.research_findings,
+                        "reference_implementations": research_results.reference_implementations,
+                        "sources": research_results.sources
+                        }
+        
+        self.writer: Agent = Agent(
+            model=self.llm,
+            instructions=[
+                "You will be provided with lots of structured outputs. Your work is to display this"
+                "in a nicely formatted manner without changing any of the content. Present all the links"
+                "as they are, with explicitly mentioned hyperlinks. Do not change any content."
+            ],
+            markdown=True,
+        )
+
+        model_response = self.writer.run(json.dumps(writer_input, indent=4), stream=True)
+        
+        return model_response
+
+    def validate_model_response(self, response: ModelResponseStatus, verbose=False) -> List[str]:
+        """Check for missing or incomplete fields in ModelResponseStatus"""
+        logger.info("Checking for missing or incomplete fields in ModelResponseStatus...", log_type="pipeline: task_analysis", console=verbose)
+        
+        missing_fields = []
+        response_dict = response.model_dump()
+
+        for field, value in response_dict.items():
+            if value == "..." or value == ["..."]:
+                missing_fields.append(field)
+            elif isinstance(value, list) and not value:
+                missing_fields.append(field)
+
+        return missing_fields
+
+    def conduct_research(self, research_prompt: str, verbose=False) -> Optional[TechnicalResearch]:
+        """Stream technical research"""
+        logger.info("Conducting technical research...", log_type="pipeline: task_analysis", console=verbose)
+
+        self.technical_researcher: Agent = Agent(
+            name="ML Technical Researcher",
+            model=self.llm,
+            description="ML Expert specialized in researching technical implementations",
+            tools=[DuckDuckGoTools(search=True, news=False)],
+            instructions=[
+                "Research and validate technical aspects of ML solutions.",
+                "1. Search for similar ML implementations and best practices",
+                "2. Find recommended models and architectures",
+                "3. Research typical hyperparameters and evaluation metrics",
+                "4. Look for implementation constraints and requirements",
+                "5. Validate technical feasibility",
+                "Provide sources for all technical information.",
+                "Focus on recent and reliable technical sources."
+            ],
+            response_model=TechnicalResearch,
+            structured_outputs=True,
+            reasoning=True,
+            # debug_mode=True,
+        )
+
+        conduct_stream = self.technical_researcher.run(research_prompt)
+        return conduct_stream.content
+
+    def finalize_analysis(self, final_prompt: str, verbose=False) -> Optional[RequirementsAnalysis]:
+        """Stream final analysis"""
+        logger.info("Finalizing analysis...", log_type="pipeline: task_analysis", console=verbose)
+
+        finalise_stream = self.requirements_analyst.run(final_prompt)
+        return finalise_stream.content
+
+    def run(self, verbose=False):
+        """
+        Run the ML analysis workflow
+        Args:
+            user_query: Description of the business problem
+        """
+        try:
+            requirements_result: Optional[RequirementsAnalysis] = self.analyze_requirements(verbose=verbose)
+
+            # logger.info("Writing initial requirements analysis...", log_type="pipeline: task_analysis", console=verbose)
+            # yield from self.write_requirements_post(requirements_result, verbose=verbose)
+
+            '''Check what needs research'''
+            missing_fields = self.validate_model_response(requirements_result.model_response, verbose=verbose)
+            logger.info("Missing fields found!", log_type="pipeline: task_analysis", console=verbose)
+            search_queries = requirements_result.search_queries
+            logger.info("Search queries found!", log_type="pipeline: task_analysis", console=verbose)
+            unclear_points = requirements_result.unclear_points
+            logger.info("Unclear points found!", log_type="pipeline: task_analysis", console=verbose)
+            
+            if missing_fields or search_queries:
+                '''Conduct technical research'''
+                logger.info("Researching technical specifications...", log_type="pipeline: task_analysis", console=verbose)
+                
+                research_prompt = (
+                    f"Research the following for this ML problem: {self.user_prompt}\n"
+                    f"Missing information needed for: {', '.join(missing_fields)}\n"
+                    f"Specific topics to research: {', '.join(search_queries)}\n"
+                    f"Points needing clarification: {', '.join(unclear_points)}\n"
+                    f"Current understanding: {requirements_result.business_understanding}"
+                )
+                logger.info("Conducting research...", log_type="pipeline: task_analysis", console=verbose)
+
+                research_result: Optional[TechnicalResearch] = self.conduct_research(research_prompt, verbose=verbose)
+                
+                # logger.info("Sharing research findings...", log_type="pipeline: task_analysis", console=verbose)
+                # research_post =  self.write_research_post(research_result, verbose=verbose)
+
+                final_prompt = (
+                    f"Original problem: {self.user_prompt}\n"
+                    f"Research findings: {research_result.research_findings}\n"
+                    "Please provide final technical specifications incorporating this research."
+                )
+
+                logger.info("Obtaining final requirements", log_type="pipeline: task_analysis", console=verbose)
+                final_result: Optional[RequirementsAnalysis] = self.finalize_analysis(final_prompt, verbose=verbose)
+
+                # logger.info("Writing final requirements...", log_type="pipeline: task_analysis", console=verbose)
+                # requirement_post = self.write_requirements_post(final_result, verbose=verbose)
+
+                return (final_result, research_result)
+            
+        except Exception as e:
+            logger.error(f"Workflow error: {str(e)}", log_type="pipeline: task_analysis", console=verbose)
+            
+
+
+
+
+        
+        

src/app/pipelines/task_analysis/ml_implementation_planner_workflow.py

@@ -0,0 +1,132 @@
+from agno.models.openai import OpenAIChat # type: ignore
+import os
+from dotenv import load_dotenv
+from src.core.utils import logger
+from typing import Optional
+from .model import RequirementsAnalysis, TechnicalResearch, ImplementationPlan
+from agno.agent import Agent, RunResponse # type: ignore
+from typing import Iterator
+import json
+
+load_dotenv()
+
+class MLImplementationPlannerWorkflow:
+    def __init__(self, requirements_analysis: RequirementsAnalysis, technical_research: Optional[TechnicalResearch] = None):
+        self.requirements_analysis = requirements_analysis
+        self.technical_research = technical_research
+        self.llm = OpenAIChat(id="gpt-4o", api_key=os.getenv('OPENAI_API_KEY'))
+
+    def create_implementation_plan(self, planning_prompt: str, verbose=False) -> Optional[ImplementationPlan]:
+        """Stream implementation plan creation"""
+        logger.info("Creating implementation plan...", log_type="pipeline: task_analysis", console=verbose)
+
+        self.architect: Agent = Agent(
+            name="ML System Architect",
+            model=self.llm,
+            description="Expert ML System Architect specialized in detailed implementation planning",
+            instructions=[
+                "Create detailed technical implementation plans for ML systems.",
+                "1. Break down the system into logical components",
+                "2. Define detailed function specifications for each component",
+                "3. Specify clear interfaces between components",
+                "4. Consider error handling and edge cases",
+                "5. Plan testing and deployment strategies",
+                "Be extremely specific about function signatures and component interactions.",
+                "Focus on maintainability and scalability in the design."
+            ],
+            response_model=ImplementationPlan,
+            structured_outputs=True,
+            reasoning=True,
+            # debug_mode=True,
+        )
+
+        planning_stream = self.architect.run(planning_prompt)
+        return planning_stream.content
+    
+    def validate_interfaces(self, validation_prompt: str, verbose=False) -> Optional[ImplementationPlan]:
+        """Stream interface validation"""
+        logger.info("Validating interfaces...", log_type="pipeline: task_analysis", console=verbose)
+        
+        architect_stream = self.architect.run(validation_prompt)
+        return architect_stream.content
+
+    def write_implementation_post(self, implementation_results: ImplementationPlan, verbose=False) -> Iterator[RunResponse]:
+        """
+        Write a blog post on a topic.
+        :param implementation_results: implementation plan results
+        :return: iterator for the workflow response
+        """
+        logger.info("Writing implementation plan...", log_type="pipeline: task_analysis", console=verbose)
+
+        writer_input = {"components": [comp.model_dump() for comp in implementation_results.components],
+                        "system_requirements": implementation_results.system_requirements,
+                        "deployment_notes": implementation_results.deployment_notes,
+                        "testing_strategy": implementation_results.testing_strategy,
+                        "implementation_order": implementation_results.implementation_order
+                        }
+        
+        self.writer: Agent = Agent(
+            model=self.llm,
+            instructions=[
+                "You will be provided with lots of structured outputs. Your work is to display this"
+                "in a nicely formatted manner without changing any of the content."
+            ],
+            markdown=True,
+        )
+
+        model_response = self.writer.run(json.dumps(writer_input, indent=4), stream=True)
+        
+        return model_response.content
+
+    def run(self, verbose=False):
+        """
+        Create implementation plan based on requirements analysis and research
+
+        Args:
+            requirements_analysis: Results from requirements analysis
+            technical_research: Optional results from technical research
+        """
+        try:
+            logger.info("Starting planning workflow...", log_type="pipeline: task_analysis", console=verbose)
+            
+            '''Prepare comprehensive prompt for the architect'''
+            planning_prompt = (
+                f"Create a detailed implementation plan for this ML system.\n\n"
+                f"Business Understanding:\n{self.requirements_analysis.business_understanding}\n\n"
+                f"Technical Specifications:\n"
+                f"- Task Type: {self.requirements_analysis.model_response.task}\n"
+                f"- Models: {', '.join(self.requirements_analysis.model_response.models)}\n"
+                f"- Data Requirements: {self.requirements_analysis.model_response.data_source}\n"
+                f"- Technical Requirements: {self.requirements_analysis.model_response.technical_requirements}\n"
+            )
+
+            if self.technical_research:
+                logger.info("Technical Research found! Modifying context...", log_type="pipeline: task_analysis", console=verbose)
+                
+                planning_prompt += (
+                    f"\nResearch Findings:\n{self.technical_research.research_findings}\n"
+                    f"Reference Implementations:\n"
+                    f"{chr(10).join(self.technical_research.reference_implementations)}"
+                )
+
+            '''Stream implementation plan'''
+            
+            logger.info("generating implementation plan...", log_type="pipeline: task_analysis", console=verbose)
+            plan_result: Optional[ImplementationPlan] = self.create_implementation_plan(planning_prompt, verbose=verbose)
+
+            if plan_result:
+                validation_prompt = (
+                    "Validate the interfaces between these components "
+                    "and ensure all dependencies are properly specified:\n"
+                    f"{plan_result.components}"
+                )
+                logger.info("validating results...", log_type="pipeline: task_analysis", console=verbose)
+
+                validate_result: Optional[ImplementationPlan] = self.validate_interfaces(validation_prompt, verbose=verbose)
+                # logger.info("writing validated implementation plan...", log_type="pipeline: task_analysis", console=verbose)
+                # final_response = self.write_implementation_post(validate_result, verbose=verbose)
+
+                return validate_result
+
+        except Exception as e:
+            logger.error("Error in planning workflow".format(e), log_type="pipeline: task_analysis", console=verbose)

src/app/pipelines/task_analysis/model.py

@@ -0,0 +1,162 @@
+from typing import List
+from typing import Iterator, List, Optional
+from enum import Enum
+from pydantic import BaseModel, Field
+
+class MLTaskType(str, Enum):
+    CLASSIFICATION = "classification"
+    REGRESSION = "regression"
+    CLUSTERING = "clustering"
+    NLP = "natural_language_processing"
+    COMPUTER_VISION = "computer_vision"
+    TIME_SERIES = "time_series"
+    ANOMALY_DETECTION = "anomaly_detection"
+    RECOMMENDATION = "recommendation"
+    OTHER = "other"
+
+
+class ModelResponseStatus(BaseModel):
+    """Technical specification for ML implementation"""
+    data_source: str = Field(
+        # default="...",
+        description="Required data sources and their characteristics"
+    )
+    data_format: str = Field(
+        # default="...",
+        description="Expected format of input data"
+    )
+    additional_data_requirement: bool = Field(
+        # default=False,
+        description="Whether additional data is needed"
+    )
+    constraints: str = Field(
+        # default="...",
+        description="Business and technical constraints"
+    )
+    task: MLTaskType = Field(
+        # default=MLTaskType.OTHER,
+        description="Type of ML task"
+    )
+    models: List[str] = Field(
+        # default=["..."],
+        description="Suggested ML models"
+    )
+    hyperparameters: List[str] = Field(
+        # default=["..."],
+        description="Key hyperparameters to consider"
+    )
+    eval_metrics: List[str] = Field(
+        # default=["..."],
+        description="Evaluation metrics for the solution"
+    )
+    technical_requirements: str = Field(
+        # default="...",
+        description="Technical implementation requirements"
+    )
+
+
+class RequirementsAnalysis(BaseModel):
+    """Initial analysis of business requirements"""
+    model_response: ModelResponseStatus
+    unclear_points: List[str] = Field(
+        default_factory=list,
+        description="Points needing clarification"
+    )
+    search_queries: List[str] = Field(
+        default_factory=list,
+        description="Topics to research"
+    )
+    business_understanding: str = Field(
+        description="Summary of business problem understanding"
+    )
+
+
+class TechnicalResearch(BaseModel):
+    """Results from technical research"""
+    model_response: ModelResponseStatus
+    research_findings: str = Field(
+        description="Key findings from research"
+    )
+    reference_implementations: List[str] = Field(
+        default_factory=list,
+        description="Similar implementation examples found"
+    )
+    sources: List[str] = Field(
+        default_factory=list,
+        description="Sources of information"
+    )
+
+
+# Implementation Planning Models
+class ComponentType(str, Enum):
+    DATA_PIPELINE = "data_pipeline"
+    PREPROCESSOR = "preprocessor"
+    MODEL = "model"
+    EVALUATOR = "evaluator"
+    INFERENCE = "inference"
+    MONITORING = "monitoring"
+    UTILITY = "utility"
+
+
+class ParameterSpec(BaseModel):
+    """Specification for a single parameter"""
+    name: str = Field(description="Name of the parameter")
+    param_type: str = Field(description="Type of the parameter")
+    description: str = Field(description="Description of the parameter")
+    default_value: str = Field(description="Default value if any")
+    required: bool = Field(description="Whether the parameter is required")
+
+
+class ConfigParam(BaseModel):
+    """Specification for a configuration parameter"""
+    name: str = Field(description="Name of the configuration parameter")
+    value_type: str = Field(description="Type of value expected")
+    description: str = Field(description="Description of the configuration parameter")
+    default: str = Field(description="Default value if any")
+
+
+class FunctionSpec(BaseModel):
+    """Detailed specification for a single function"""
+    name: str = Field(description="Name of the function")
+    description: str = Field(description="Detailed description of function's purpose")
+    input_params: List[ParameterSpec] = Field(
+        description="List of input parameters and their specifications"
+    )
+    return_type: str = Field(description="Return type and description")
+    dependencies: List[str] = Field(
+        description="Required dependencies/imports"
+    )
+    error_handling: List[str] = Field(
+        description="Expected errors and handling strategies"
+    )
+
+
+class ComponentSpec(BaseModel):
+    """Specification for a component (module) of the system"""
+    name: str = Field(description="Name of the component")
+    type: ComponentType = Field(description="Type of component")
+    description: str = Field(description="Detailed description of component's purpose")
+    functions: List[FunctionSpec] = Field(description="Functions within this component")
+    dependencies: List[str] = Field(
+        description="External package dependencies"
+    )
+    config_params: List[ConfigParam] = Field(
+        description="Configuration parameters needed"
+    )
+
+
+class ImplementationPlan(BaseModel):
+    """Complete implementation plan for the ML system"""
+    components: List[ComponentSpec] = Field(description="System components")
+    system_requirements: List[str] = Field(
+        description="System-level requirements and dependencies"
+    )
+    deployment_notes: str = Field(
+        description="Notes on deployment and infrastructure"
+    )
+    testing_strategy: str = Field(
+        description="Strategy for testing components"
+    )
+    implementation_order: List[str] = Field(
+        description="Suggested order of implementation"
+    )

src/app/schemas/requests/__init__.py

@@ -0,0 +1 @@
+from .task_analysis import TaskAnalysisRequestSchema

src/app/schemas/requests/eda.py

@@ -0,0 +1,25 @@
+from pydantic import BaseModel, Field
+from src.app.pipelines.task_analysis.model import ImplementationPlan, RequirementsAnalysis, TechnicalResearch
+from typing import Optional
+
+class EdaRequestSchema(BaseModel):
+    dataset_path: str = Field(
+        default=None,
+        description="Path of the dataset stored locally"
+    )
+    user_prompt: str = Field(
+        default=None,
+        description="Contains the initial prompt given by the user"
+    )
+    requirement_analysis: Optional[RequirementsAnalysis] = Field(
+        default=None,
+        description="Contains the analysis of the user task/prompt"
+    )
+    technical_research: Optional[TechnicalResearch] = Field(
+        default=None,
+        description="Contains the technical research of the user task/prompt"
+    )
+    implementation_plan: Optional[ImplementationPlan] = Field(
+        default=None,
+        description="Includes the detailed plan for what steps to take"
+    )

src/app/schemas/requests/task_analysis.py

@@ -0,0 +1,12 @@
+from pydantic import BaseModel, Field
+
+class TaskAnalysisRequestSchema(BaseModel):
+    '''Schema for accepting user instructions/prompts/task and dataset (Currently only supporting CSV)'''
+    user_prompt: str = Field(
+        default = None,
+        description = "Defines the user prompt/instructions/task"
+    )
+    file_name: str = Field(
+        default = None,
+        description = "Contains the filename of the dataset. Stored in a temporary storage"
+    )

src/app/schemas/responses/eda.py

@@ -0,0 +1,31 @@
+from pydantic import BaseModel, Field
+from typing import List
+from src.app.pipelines.eda.agents.models import OrchestratorAgentResponseSchema, ExecuterAgentResponseSchema, AnalyzerAgentResponseSchema, JudgingAgentResponseSchema
+
+class IterationDetails(BaseModel):
+    iteration_number: int = Field(
+        default=None,
+        description="Contains the iteration number"
+    )
+    orchestrator_response: OrchestratorAgentResponseSchema = Field(
+        default=None,
+        description="Contains orchestrator agent's response for this iteration"
+    )
+    executer_response: ExecuterAgentResponseSchema = Field(
+        default=None,
+        description="Contains executer agent's response for this iteration"
+    )
+    analyzer_response: AnalyzerAgentResponseSchema = Field(
+        default=None,
+        description="Contains analyzer agent's response for this iteration"
+    )
+    judge_response: JudgingAgentResponseSchema = Field(
+        default=None,
+        description="Contains judging agent's response for this iteration"
+    )
+    
+class IterationLogs(BaseModel):
+    logs: List[IterationDetails] = Field(
+        default=None,
+        description="Contains a list of logs for each iteration"
+    )

src/core/cache/redis_cache.py

@@ -0,0 +1,33 @@
+import json
+import redis.asyncio as redis  # type: ignore
+from typing import Any, Optional
+from src.core.utils import read_config
+
+
+class RedisCache:
+    '''Reads the server settings from config file. Can import from anywhere'''
+    def __init__(self):
+        settings = read_config(config_path="config.yaml")
+
+        self._client = redis.Redis(
+            host = settings['redis_server']['host'],
+            port = settings['redis_server']['port'],
+            db = settings['redis_server']['db'],
+            decode_responses = True
+        )
+    async def set(self, key: str, value: Any)->None:
+        await self._client.set(key, json.dumps(value))
+
+    async def get(self, key: str)-> Optional[Any]:
+        value = await self._client.get(key)
+        if value:
+            return json.loads(value)
+        return None
+    
+    async def delete(self, key: str)->None:
+        await self._client.delete(key)
+    
+    async def clear(self)->None:
+        await self._client.flushdb()
+
+cache = RedisCache()

src/core/server.py

@@ -0,0 +1,23 @@
+from fastapi import FastAPI
+from src.api import router
+from src.core.utils import logger, read_config
+
+def init_routers(app_: FastAPI) -> None:
+    app_.include_router(router)
+
+def create_app() -> FastAPI:
+    config = read_config("config.yaml")
+
+    app_ = FastAPI(
+        title="Franky API",
+        description="In Development",
+        version="1.0.0",
+    )
+
+    app_.state.config = config
+
+    init_routers(app_=app_)
+    logger.info("Server started successfully", log_type="server", console=True) 
+    return app_
+
+app = create_app()

src/core/utils/__init__.py

@@ -0,0 +1,3 @@
+from .read_config import read_config
+from .logger import logger
+from .knowledge_base import KnowledgeBaseClass

src/core/utils/knowledge_base.py

@@ -0,0 +1,81 @@
+import os
+import chromadb
+from typing import List
+from agno.agent import Agent
+from dotenv import load_dotenv
+from pydantic import BaseModel, Field
+from agno.models.openai import OpenAIChat
+from llama_index.core import StorageContext
+from llama_index.core import VectorStoreIndex
+from llama_index.core.retrievers import VectorIndexRetriever
+from agno.knowledge.llamaindex import LlamaIndexKnowledgeBase
+from llama_index.vector_stores.chroma import ChromaVectorStore
+from .prompts import missing_value_analysis_agent_desc, missing_value_analysis_agent_instructions
+
+load_dotenv()
+
+class ResponseSchema(BaseModel):
+    code_generated: str = Field(..., description="Python code generated for the identified statistical tests.")
+    libraries_necessary: List[str] = Field(..., description="List of necessary Python libraries with preferred versions (e.g., ['pandas>=1.5.0', 'numpy>=1.24.0', 'scipy>=1.10.0']).")
+    reasoning: str = Field(..., description="Detailed reasoning for choosing the specific statistical tests based on the input report and the knowledge base.")
+
+class KnowledgeBaseClass:
+    def __init__(self)->None:
+        
+        self.knowlede_base_map = {
+            "classification": {
+                "raw_data_path": r"knowledge_base\raw\classification",
+                "vector_index_path": r"knowledge_base\vector\classification",
+                "collection_name": "classification_db"
+            },
+            "regression": {
+                "raw_data_path": r"knowledge_base\raw\regression",
+                "vector_index_path": r"knowledge_base\vector\regression",
+                "collection_name": "regression_db"
+            },
+            "time_series": {
+                "raw_data_path": r"knowledge_base\raw\time_series",
+                "vector_index_path": r"knowledge_base\vector\time_series",
+                "collection_name": "time_series_db"
+            }
+        }
+
+        self.agent_map = {
+            'missing_value_analysis_agent': {
+                'description': missing_value_analysis_agent_desc,
+                'instructions': missing_value_analysis_agent_instructions
+            } 
+        }
+
+    def initialize_knowledge_base(self, task_type: str)->LlamaIndexKnowledgeBase:
+        selected_knowledge_base = self.knowlede_base_map[task_type]
+        db = chromadb.PersistentClient(path=selected_knowledge_base['vector_index_path'])
+        chroma_collection = db.get_or_create_collection(selected_knowledge_base['collection_name'])
+        vector_store = ChromaVectorStore(chroma_collection=chroma_collection)
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+        index = VectorStoreIndex.from_vector_store(vector_store, storage_context=storage_context)
+        retriever = VectorIndexRetriever(index)
+        knowledge_base = LlamaIndexKnowledgeBase(retriever=retriever)
+    
+        return knowledge_base
+
+    def initialize_agent(self, agent_name: str, llm_choice: str, knowledge_base: LlamaIndexKnowledgeBase)->Agent:
+        selected_agent = self.agent_map[agent_name]
+        llm = OpenAIChat(id=llm_choice, api_key=os.getenv('OPENAI_API_KEY'))
+
+        agent = Agent(
+            model = llm, 
+            description=selected_agent['description'], 
+            instructions=selected_agent['instructions'], 
+            knowledge=knowledge_base, 
+            search_knowledge=True, 
+            response_model=ResponseSchema
+        )
+
+        return agent
+
+    
+
+
+
+        

src/core/utils/logger.py

@@ -0,0 +1,46 @@
+import logging
+from datetime import datetime
+
+class LogManager:
+    def __init__(self, log_file_path: str):
+        self.logger = logging.getLogger("LogManager")
+        self.logger.setLevel(logging.DEBUG)
+
+        if not self.logger.handlers:
+            formatter = logging.Formatter('%(asctime)s - %(levelname)s - [%(log_type)s] - %(message)s')
+
+            file_handler = logging.FileHandler(log_file_path, encoding='utf-8')
+            file_handler.setFormatter(formatter)
+            file_handler.setLevel(logging.DEBUG)
+            self.logger.addHandler(file_handler)
+
+            self.console_handler = logging.StreamHandler()
+            self.console_handler.setFormatter(formatter)
+            self.console_handler.setLevel(logging.DEBUG)
+
+        else:
+            self.console_handler = None        
+    
+    def log(self, level, message, log_type: str, console=False):
+        extra = {'log_type': log_type}
+
+        if console and self.console_handler:
+            self.logger.addHandler(self.console_handler)
+
+        self.logger.log(level, message, extra=extra)
+
+        if console and self.console_handler:
+            self.logger.removeHandler(self.console_handler)
+
+    def info(self, message, log_type: str, console=False):
+        self.log(logging.INFO, message, log_type, console)
+
+    def error(self, message, log_type: str, console=False):
+        self.log(logging.ERROR, message, log_type, console)
+
+    def debug(self, message, log_type: str, console=False):
+        self.log(logging.DEBUG, message, log_type, console)
+
+
+timestamp = datetime.now().strftime("%Y-%m-%d_%H")
+logger = LogManager(log_file_path=f"src/core/logs/log_{timestamp}.log")