fixed errors

src/api/v1/eda_engine/data_quality.py

@@ -1,7 +1,9 @@
 import os
+import math
 import shutil
 from fastapi import APIRouter
 from src.core.utils import logger
+from fastapi.responses import JSONResponse
 from fastapi import APIRouter, UploadFile, File, HTTPException, Form
 from src.app.pipelines.modules import DataQualityAssessmentWorkflow
 
@@ -13,6 +15,17 @@ def delete_dir_contents(directory: str)->None:
         if os.path.isfile(file_path):
             os.remove(file_path)
 
+def sanitize_for_json(data):
+    if isinstance(data, dict):
+        return {k: sanitize_for_json(v) for k, v in data.items()}
+    elif isinstance(data, list):
+        return [sanitize_for_json(v) for v in data]
+    elif isinstance(data, float):
+        if math.isinf(data) or math.isnan(data):
+            return None
+        return data
+    return data
+
 @data_quality_router.post('/')
 async def main(file: UploadFile = File(...), ml_task: str = Form(None)):
     ''' ## This endpoint accepts a CSV file upload  to initiate the Data Quality Workflow. 
@@ -46,10 +59,9 @@ async def main(file: UploadFile = File(...), ml_task: str = Form(None)):
         ds_wf = DataQualityAssessmentWorkflow(data_source=f'{downloads_path}/dataset.csv', llm_choice="gpt-4o-mini", ml_task=ml_task)
         results = ds_wf.run(verbose=True)
 
-        return {
-            "status": "Pipeline finished running",
-            "results": results
-        }
+        sanitized_data = sanitize_for_json(results)
+
+        return JSONResponse(content=sanitized_data)
 
     except Exception as e:
         logger.error(f"DataQualityAssessmentWorkflow failed with error: {e}", log_type='eda-engine/data_quality', console=True)

src/api/v1/eda_engine/data_statistics.py

@@ -1,12 +1,26 @@
 import os
+import math
 import shutil
 from fastapi import APIRouter
 from src.core.utils import logger
-from fastapi import APIRouter, UploadFile, File, HTTPException, Form
+from fastapi.responses import JSONResponse
 from src.app.pipelines.modules import DataStatisticsWorkflow
+from fastapi import APIRouter, UploadFile, File, HTTPException, Form
+
 
 data_statistics_router = APIRouter()
 
+def sanitize_for_json(data):
+    if isinstance(data, dict):
+        return {k: sanitize_for_json(v) for k, v in data.items()}
+    elif isinstance(data, list):
+        return [sanitize_for_json(v) for v in data]
+    elif isinstance(data, float):
+        if math.isinf(data) or math.isnan(data):
+            return None
+        return data
+    return data
+
 def delete_dir_contents(directory: str)->None:
     for filename in os.listdir(directory):
         file_path = os.path.join(directory, filename)
@@ -47,10 +61,10 @@ async def main(file: UploadFile = File(...),  ml_task: str = Form(None)):
         ds_wf = DataStatisticsWorkflow(data_source=f'{downloads_path}/dataset.csv', llm_choice="gpt-4o-mini", ml_task=ml_task)
         results = ds_wf.run(verbose=True)
 
-        return {
-            "status": "Pipeline finished running",
-            "results": results
-        }
+        sanitized_data = sanitize_for_json(results)
+
+        return JSONResponse(content=sanitized_data)
+
 
     except Exception as e:
         logger.error(f"DataStatisticsWorkflow failed with error: {e}", log_type='eda-engine/data_statistics', console=True)

src/api/v1/eda_engine/univariate_analysis.py

@@ -1,7 +1,8 @@
 import os
+import math
 import shutil
-from typing import Optional
 from src.core.utils import logger
+from fastapi.responses import JSONResponse
 from fastapi import APIRouter, UploadFile, File, HTTPException, Form
 from src.app.pipelines.modules import UnivariateAnalysisWorkflow
 
@@ -13,6 +14,17 @@ def delete_dir_contents(directory: str)->None:
         if os.path.isfile(file_path):
             os.remove(file_path)
 
+def sanitize_for_json(data):
+    if isinstance(data, dict):
+        return {k: sanitize_for_json(v) for k, v in data.items()}
+    elif isinstance(data, list):
+        return [sanitize_for_json(v) for v in data]
+    elif isinstance(data, float):
+        if math.isinf(data) or math.isnan(data):
+            return None
+        return data
+    return data
+
 @univariate_analysis_router.post('/')
 async def main(file: UploadFile = File(...), ml_task: str = Form(None)):
     ''' ## This endpoint accepts a CSV file upload  to initiate the Univarite Analysis Workflow. 
@@ -47,10 +59,9 @@ async def main(file: UploadFile = File(...), ml_task: str = Form(None)):
         ua_wf = UnivariateAnalysisWorkflow(data_source=f'{downloads_path}/dataset.csv', llm_choice="gpt-4o-mini", ml_task=ml_task)
         results = ua_wf.run(verbose=True)
 
-        return {
-            "status": "Pipeline finished running",
-            "results": results
-        }
+        sanitized_data = sanitize_for_json(results)
+
+        return JSONResponse(content=sanitized_data)
 
     except Exception as e:
         logger.error(f"UnivariateAnalysisWorkflow failed with error: {e}", log_type='eda-engine/dataunivariate_analysis_statistics', console=True)

src/app/pipelines/modules/data_quality_assessment.py

@@ -1632,21 +1632,21 @@ class DataQualityAssessmentWorkflow:
         }
 
         try:
-            mva_str = json.dumps(mva_results, indent=2, default=str)
+            mva_str = json.dumps(mva_results, indent=2, default=str, allow_nan=True)
             final_result['missing_value_analysis']['report'] = self.generate_report_from_agent(mva_str)
         except:
             logger.error("Failed to generate report for mva....", log_type='data_quality_assessment', console=verbose)
             pass
 
         try:
-            duplicate_analysis_results_str = json.dumps(duplicate_analysis_results, indent=2, default=str)
+            duplicate_analysis_results_str = json.dumps(duplicate_analysis_results, indent=2, default=str, allow_nan=True)
             final_result['duplicate_analysis']['report'] = self.generate_report_from_agent(duplicate_analysis_results_str)
         except:
             logger.error("Failed to generate report for duplicate analysis....", log_type='data_quality_assessment', console=verbose)
             pass
 
         try:
-            data_quality_results_str = json.dumps(data_quality_results, indent=2, default=str)
+            data_quality_results_str = json.dumps(data_quality_results, indent=2, default=str, allow_nan=True)
             data_quality_results_str = data_quality_results_str +'\n'+data_quality_summary
             final_result['data_quality_analysis']['report'] = self.generate_report_from_agent(data_quality_results_str)
         except:

src/app/pipelines/modules/data_statistics.py

@@ -71,7 +71,7 @@ class DataStatisticsWorkflow:
 
         serializable_results = process_dict(results)
 
-        return json.dumps(serializable_results, indent=indent)
+        return json.dumps(serializable_results, indent=indent, allow_nan=True)
 
     def build_statistical_summary(self, data_source: str = None, verbose=False) -> Dict[str, Any]:
         '''Get the basic central tendancy, dispersion, quantiles, distinct values, frequency distributions and sparsity'''

src/app/pipelines/task_analysis/ml_analysis_workflow.py

@@ -138,7 +138,7 @@ class MLAnalysisWorkflow:
             try:
                 return future.result(timeout=30)
             except concurrent.futures.TimeoutError:
-                logger.warning("Technical research timed out after 30 seconds.", log_type="pipeline: timeout", console=verbose)
+                logger.info("Technical research timed out after 30 seconds.", log_type="pipeline: timeout", console=verbose)
                 return None
 
     def finalize_analysis(self, final_prompt: str, verbose=False) -> Optional[RequirementsAnalysis]: