feat: Implement revenue data organization workflow with JSON output

- Added a new revenue data organization task in `data_arrangement_1.txt` to structure extracted financial data for Excel reporting.
- Introduced `RestrictedPythonTools` for self-healing Python execution with directory constraints and package management.
- Created `RestrictedShellTools` to execute shell commands within a specified base directory, enhancing security and preventing directory traversal.
- Developed `FinancialDocumentWorkflow` to manage the entire financial document analysis process, including data extraction, arrangement, and Excel report generation.
- Integrated session management for handling input/output directories and caching extraction results.
- Enhanced error handling and logging throughout the workflow for better traceability and debugging.

.claude/settings.local.json

@@ -0,0 +1,13 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(python3:*)",
+      "Bash(mkdir:*)",
+      "Bash(ls:*)",
+      "Bash(find:*)",
+      "Bash(grep:*)",
+      "Bash(python test_prompt_loading.py:*)"
+    ],
+    "deny": []
+  }
+}

app.py

@@ -8,7 +8,7 @@ os.environ.setdefault("MPLCONFIGDIR", "/tmp/mpl_cache")
 import logging
 from pathlib import Path
 import uuid
-from workflow.financial_workflow import FinancialDocumentWorkflow
+from workflow.financial_workflow_working import FinancialDocumentWorkflow
 from agno.storage.sqlite import SqliteStorage
 from utils.file_handler import FileHandler
 from config.settings import settings
@@ -1892,7 +1892,7 @@ def create_gradio_app():
                     logger.info("Backend: Starting Step 1 - Data Extraction")
                     
                     # Run the workflow and track progress
-                    result = ui.workflow.run_workflow()
+                    result = list(ui.workflow.run(file_path=ui.workflow.file_path))
                     progress_state['result'][0] = result
                     
                     # Signal completion
@@ -1941,8 +1941,9 @@ def create_gradio_app():
             if progress_state['error'][0]:
                 raise progress_state['error'][0]
             
-            workflow_response = progress_state['result'][0]
-            workflow_results = workflow_response.content
+            workflow_responses = progress_state['result'][0]
+            # Extract content from all responses and join them
+            workflow_results = "\n".join([response.content for response in workflow_responses])
             
             # The workflow has completed all steps - just display the results
             logger.info("📊 Displaying workflow results")

config/settings.py

@@ -1,114 +1,237 @@
+"""
+Configuration settings for Data Extractor Using Gemini
+Optimized for Gemini-only model usage with robust directory management
+"""
+
 import os
 from pathlib import Path
 from dotenv import load_dotenv
+import logging
 
+# Load environment variables
 load_dotenv()
 
+logger = logging.getLogger(__name__)
+
 
 class Settings:
+    """Configuration settings with Gemini-only model support and robust directory management."""
+    
+    # === GEMINI MODEL CONFIGURATION ===
     GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
-    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
-    MAX_FILE_SIZE_MB = 50
+    
+    # Gemini model specifications - using gemini-2.5-flash (supports thinking budget)
+    DATA_EXTRACTOR_MODEL = os.getenv("DATA_EXTRACTOR_MODEL", "gemini-2.5-pro")
+    DATA_ARRANGER_MODEL = os.getenv("DATA_ARRANGER_MODEL", "gemini-2.5-pro") 
+    CODE_GENERATOR_MODEL = os.getenv("CODE_GENERATOR_MODEL", "gemini-2.5-flash")
+    
+    # Thinking budgets optimized for each task type
+    DATA_EXTRACTOR_MODEL_THINKING_BUDGET = int(os.getenv("DATA_EXTRACTOR_THINKING_BUDGET", "4096"))
+    DATA_ARRANGER_MODEL_THINKING_BUDGET = int(os.getenv("DATA_ARRANGER_THINKING_BUDGET", "4096"))
+    CODE_GENERATOR_MODEL_THINKING_BUDGET = int(os.getenv("CODE_GENERATOR_THINKING_BUDGET", "4096"))
+    
+    # === FILE PROCESSING CONFIGURATION ===
+    MAX_FILE_SIZE_MB = int(os.getenv("MAX_FILE_SIZE_MB", "50"))
     SUPPORTED_FILE_TYPES = [
-        "pdf",
-        "txt",
-        "png",
-        "jpg",
-        "jpeg",
-        "docx",
-        "xlsx",
-        "csv",
-        "md",
-        "json",
-        "xml",
-        "html",
-        "py",
-        "js",
-        "ts",
-        "doc",
-        "xls",
-        "ppt",
-        "pptx",
+        "pdf", "txt", "docx", "xlsx", "csv", "md", "json", "xml", "html",
+        "png", "jpg", "jpeg", "doc", "xls", "ppt", "pptx"
     ]
-    # Use /tmp for temporary files on Hugging Face Spaces (or override with TEMP_DIR env var)
-    TEMP_DIR = Path(os.getenv("TEMP_DIR", "/tmp/data_extractor_temp"))
-    DOCKER_IMAGE = os.getenv("DOCKER_IMAGE", "python:3.12-slim")
-    COORDINATOR_MODEL = os.getenv("COORDINATOR_MODEL", "gemini-2.5-pro")
-    PROMPT_ENGINEER_MODEL = os.getenv("PROMPT_ENGINEER_MODEL", "gemini-2.5-pro")
-    DATA_EXTRACTOR_MODEL = os.getenv("DATA_EXTRACTOR_MODEL", "gemini-2.5-pro")
-    DATA_ARRANGER_MODEL = os.getenv("DATA_ARRANGER_MODEL", "gemini-2.5-pro")
-    CODE_GENERATOR_MODEL = os.getenv("CODE_GENERATOR_MODEL", "gemini-2.0-flash")
-
-    COORDINATOR_MODEL_THINKING_BUDGET=2048
-    PROMPT_ENGINEER_MODEL_THINKING_BUDGET=2048
-    DATA_EXTRACTOR_MODEL_THINKING_BUDGET=-1
-    DATA_ARRANGER_MODEL_THINKING_BUDGET=3072
-    CODE_GENERATOR_MODEL_THINKING_BUDGET=3072
-
+    
+    # === DIRECTORY MANAGEMENT ===
+    # Centralized working directory - all operations happen within this directory
+    WORKING_DIR = Path(os.getenv("WORKING_DIR", "/tmp/data_extractor_gemini"))
+    
+    # Subdirectories within working directory
+    TEMP_DIR = WORKING_DIR / "temp"
+    INPUT_DIR = WORKING_DIR / "input" 
+    OUTPUT_DIR = WORKING_DIR / "output"
+    CACHE_DIR = WORKING_DIR / "cache"
+    LOGS_DIR = WORKING_DIR / "logs"
+    
+    # === WORKFLOW CONFIGURATION ===
+    # Retry and timeout settings
+    MAX_RETRIES = int(os.getenv("MAX_RETRIES", "3"))
+    RETRY_DELAY_SECONDS = int(os.getenv("RETRY_DELAY_SECONDS", "5"))
+    AGENT_TIMEOUT_SECONDS = int(os.getenv("AGENT_TIMEOUT_SECONDS", "300"))
+    
+    # Cache settings
+    ENABLE_CACHING = os.getenv("ENABLE_CACHING", "true").lower() == "true"
+    CACHE_TTL_HOURS = int(os.getenv("CACHE_TTL_HOURS", "24"))
+    
+    @classmethod
+    def initialize_directories(cls):
+        """Initialize all required directories with proper permissions."""
+        directories = [
+            cls.WORKING_DIR,
+            cls.TEMP_DIR,
+            cls.INPUT_DIR,
+            cls.OUTPUT_DIR,
+            cls.CACHE_DIR,
+            cls.LOGS_DIR
+        ]
+        
+        created_dirs = []
+        for directory in directories:
+            try:
+                directory.mkdir(parents=True, exist_ok=True)
+                
+                # Test write permissions
+                test_file = directory / ".write_test"
+                test_file.write_text("test")
+                test_file.unlink()
+                
+                created_dirs.append(str(directory))
+                logger.debug(f"Directory initialized: {directory}")
+                
+            except Exception as e:
+                logger.error(f"Failed to initialize directory {directory}: {e}")
+                raise RuntimeError(f"Cannot create or write to directory {directory}: {e}")
+        
+        logger.info(f"Successfully initialized {len(created_dirs)} directories")
+        return created_dirs
+    
     @classmethod
     def validate_config(cls):
-        """Validate configuration and create necessary directories."""
+        """Comprehensive configuration validation with detailed error reporting."""
         errors = []
         warnings = []
         
-        # Check required API keys
+        # === CRITICAL VALIDATIONS ===
+        
+        # Google API Key validation
         if not cls.GOOGLE_API_KEY:
-            errors.append("GOOGLE_API_KEY is required - get it from Google AI Studio")
-            
-        # Check for optional but recommended API keys
-        openai_key = os.getenv("OPENAI_API_KEY")
-        if not openai_key:
-            warnings.append("OPENAI_API_KEY not set - OpenAI models will not be available")
-            
-        # Validate and create temp directory
+            errors.append("GOOGLE_API_KEY is required. Get it from https://aistudio.google.com/app/apikey")
+        elif len(cls.GOOGLE_API_KEY) < 30:
+            warnings.append("GOOGLE_API_KEY appears to be too short - verify it's correct")
+        
+        # Model name validation
+        gemini_models = [cls.DATA_EXTRACTOR_MODEL, cls.DATA_ARRANGER_MODEL, cls.CODE_GENERATOR_MODEL]
+        for i, model in enumerate(gemini_models):
+            model_names = ["DATA_EXTRACTOR_MODEL", "DATA_ARRANGER_MODEL", "CODE_GENERATOR_MODEL"]
+            if not model:
+                errors.append(f"{model_names[i]} cannot be empty")
+            elif not model.startswith("gemini-"):
+                errors.append(f"{model_names[i]} must be a Gemini model (starts with 'gemini-'), got: {model}")
+        
+        # Directory validation
         try:
-            cls.TEMP_DIR.mkdir(exist_ok=True, parents=True)
-            # Test write permissions
-            test_file = cls.TEMP_DIR / ".write_test"
-            try:
-                test_file.write_text("test")
-                test_file.unlink()
-            except Exception as e:
-                errors.append(f"Cannot write to temp directory {cls.TEMP_DIR}: {e}")
+            cls.initialize_directories()
         except Exception as e:
-            errors.append(f"Cannot create temp directory {cls.TEMP_DIR}: {e}")
-            
-        # Validate file size limits
+            errors.append(f"Directory initialization failed: {e}")
+        
+        # === MODERATE VALIDATIONS ===
+        
+        # File size validation
         if cls.MAX_FILE_SIZE_MB <= 0:
             errors.append("MAX_FILE_SIZE_MB must be positive")
         elif cls.MAX_FILE_SIZE_MB > 100:
-            warnings.append(f"MAX_FILE_SIZE_MB ({cls.MAX_FILE_SIZE_MB}) is very large")
-            
-        # Validate supported file types
+            warnings.append(f"MAX_FILE_SIZE_MB ({cls.MAX_FILE_SIZE_MB}) is very large - may cause memory issues")
+        
+        # Supported file types validation
         if not cls.SUPPORTED_FILE_TYPES:
             errors.append("SUPPORTED_FILE_TYPES cannot be empty")
-            
-        # Validate model names
-        model_fields = ['DATA_EXTRACTOR_MODEL', 'DATA_ARRANGER_MODEL', 'CODE_GENERATOR_MODEL']
-        for field in model_fields:
-            model_name = getattr(cls, field)
-            if not model_name:
-                errors.append(f"{field} cannot be empty")
-            elif not model_name.startswith(('gemini-', 'gpt-', 'claude-')):
-                warnings.append(f"{field} '{model_name}' may not be a valid model name")
-                
-        # Return validation results
+        
+        # Thinking budget validation
+        budgets = [
+            (cls.DATA_EXTRACTOR_MODEL_THINKING_BUDGET, "DATA_EXTRACTOR_MODEL_THINKING_BUDGET"),
+            (cls.DATA_ARRANGER_MODEL_THINKING_BUDGET, "DATA_ARRANGER_MODEL_THINKING_BUDGET"),
+            (cls.CODE_GENERATOR_MODEL_THINKING_BUDGET, "CODE_GENERATOR_MODEL_THINKING_BUDGET")
+        ]
+        
+        for budget, name in budgets:
+            if budget < 1024:
+                warnings.append(f"{name} ({budget}) is quite low - may affect model performance")
+            elif budget > 8192:
+                warnings.append(f"{name} ({budget}) is very high - may be unnecessary")
+        
+        # Retry configuration validation
+        if cls.MAX_RETRIES < 1:
+            warnings.append("MAX_RETRIES should be at least 1")
+        elif cls.MAX_RETRIES > 10:
+            warnings.append("MAX_RETRIES is very high - may cause long delays")
+        
+        # === RESULT PROCESSING ===
+        
         if errors:
-            error_msg = "Configuration validation failed:\n" + "\n".join(f"- {error}" for error in errors)
+            error_msg = "❌ Configuration validation failed:\n"
+            error_msg += "\n".join(f"  • {error}" for error in errors)
+            
             if warnings:
-                error_msg += "\n\nWarnings:\n" + "\n".join(f"- {warning}" for warning in warnings)
-            raise ValueError(error_msg)
+                error_msg += "\n\n⚠️  Warnings:\n"
+                error_msg += "\n".join(f"  • {warning}" for warning in warnings)
             
+            raise ValueError(error_msg)
+        
         if warnings:
-            import logging
-            logger = logging.getLogger(__name__)
-            logger.warning("Configuration warnings:\n" + "\n".join(f"- {warning}" for warning in warnings))
-            
+            logger.warning("Configuration warnings detected:")
+            for warning in warnings:
+                logger.warning(f"  • {warning}")
+        
+        logger.info("✅ Configuration validation successful")
         return True
     
+    @classmethod
+    def get_session_directories(cls, session_id: str):
+        """Get session-specific directory structure."""
+        session_base = cls.WORKING_DIR / session_id
+        
+        return {
+            "base": session_base,
+            "input": session_base / "input",
+            "output": session_base / "output", 
+            "temp": session_base / "temp",
+            "cache": session_base / "cache"
+        }
+    
+    @classmethod
+    def create_session_directories(cls, session_id: str):
+        """Create and validate session-specific directories."""
+        session_dirs = cls.get_session_directories(session_id)
+        
+        created = []
+        for name, directory in session_dirs.items():
+            try:
+                directory.mkdir(parents=True, exist_ok=True)
+                
+                # Test write permissions
+                test_file = directory / ".write_test"
+                test_file.write_text("test")
+                test_file.unlink()
+                
+                created.append(str(directory))
+                
+            except Exception as e:
+                logger.error(f"Failed to create session directory {name}: {e}")
+                raise RuntimeError(f"Cannot create session directory {directory}: {e}")
+        
+        logger.info(f"Created {len(created)} session directories for {session_id}")
+        return session_dirs
+    
+    @classmethod
+    def cleanup_session(cls, session_id: str, keep_output: bool = True):
+        """Clean up session directories with option to preserve output."""
+        session_dirs = cls.get_session_directories(session_id)
+        
+        import shutil
+        cleaned = []
+        
+        for name, directory in session_dirs.items():
+            if keep_output and name == "output":
+                continue
+                
+            if directory.exists():
+                try:
+                    shutil.rmtree(directory)
+                    cleaned.append(str(directory))
+                except Exception as e:
+                    logger.warning(f"Could not clean {name} directory: {e}")
+        
+        logger.info(f"Cleaned {len(cleaned)} session directories for {session_id}")
+        return cleaned
+    
     @classmethod
     def get_debug_info(cls):
-        """Get debug information about current configuration."""
+        """Get comprehensive debug information about current configuration."""
         import platform
         import sys
         
@@ -117,16 +240,26 @@ class Settings:
             "platform": platform.platform(),
             "temp_dir": str(cls.TEMP_DIR),
             "temp_dir_exists": cls.TEMP_DIR.exists(),
-            "supported_file_types": len(cls.SUPPORTED_FILE_TYPES),
-            "max_file_size_mb": cls.MAX_FILE_SIZE_MB,
-            "has_google_api_key": bool(cls.GOOGLE_API_KEY),
-            "has_openai_api_key": bool(os.getenv("OPENAI_API_KEY")),
             "models": {
                 "data_extractor": cls.DATA_EXTRACTOR_MODEL,
-                "data_arranger": cls.DATA_ARRANGER_MODEL, 
-                "code_generator": cls.CODE_GENERATOR_MODEL
+                "data_arranger": cls.DATA_ARRANGER_MODEL,
+                "code_generator": cls.CODE_GENERATOR_MODEL,
+            },
+            "api_keys": {
+                "google_api_key_present": bool(cls.GOOGLE_API_KEY),
+                "google_api_key_length": len(cls.GOOGLE_API_KEY) if cls.GOOGLE_API_KEY else 0
             }
         }
 
 
+# Global settings instance
 settings = Settings()
+
+# Auto-initialize directories on import
+try:
+    settings.initialize_directories()
+    logger.debug("Settings initialized successfully")
+except Exception as e:
+    logger.error(f"Failed to initialize settings: {e}")
+    # Don't raise here to allow import to succeed
+

instructions/agents/code_generator.json

@@ -1,83 +1,249 @@
 {
-    "instructions": [
-      "=== EXCEL REPORT GENERATION SPECIALIST ===",
-      "You are an Excel report generator that creates formatted workbooks from financial data.",
-      "",
-      "CRITICAL: Always use these exact steps in order - FAILURE IS NOT ACCEPTABLE:",
-      "",
-      "=== MANDATORY STEP-BY-STEP EXECUTION ===",
-      "STEP 1: Install required packages",
-      "- MUST execute: run_shell_command('pip install openpyxl pandas xlsxwriter')",
-      "- Verify installation: run_shell_command('pip list | grep openpyxl')",
-      "- If fails, try: run_shell_command('python -m pip install openpyxl pandas xlsxwriter')",
-      "",
-      "STEP 2: Read the arranged_financial_data.json file",
-      "- MUST use: read_file('arranged_financial_data.json')",
-      "- Analyze structure completely before proceeding",
-      "- Log all keys and data types found",
-      "",
-      "STEP 3: Create Excel file with proper error handling",
-      "- Create generate_excel_report.py with comprehensive try-catch blocks",
-      "- Include progress logging for each worksheet creation",
-      "- Handle missing data gracefully with default values",
-      "- Use professional formatting (headers, colors, borders)",
-      "",
-      "STEP 4: Execute the Python script",
-      "- MUST execute: run_shell_command('python generate_excel_report.py 2>&1')",
-      "- Capture and log all output and errors",
-      "- If execution fails, debug and retry with fixes",
-      "",
-      "STEP 5: Test that the file was created successfully",
-      "- MUST verify: run_shell_command('ls -la *.xlsx') or run_shell_command('dir *.xlsx')",
-      "- Check file size > 5KB: run_shell_command('du -h *.xlsx')",
-      "- Test file integrity: attempt to open with pandas",
-      "",
-      "STEP 6: Return the full file path of the created Excel file",
-      "- Use absolute path resolution",
-      "- Confirm file exists and is accessible",
-      "- Log final success message with file details",
-      "",
-      "=== ERROR HANDLING REQUIREMENTS ===",
-      "Handle all errors gracefully and provide clear error messages:",
-      "- Package installation failures: try alternative methods",
-      "- JSON parsing errors: provide specific line/key information",
-      "- Excel creation errors: log exact openpyxl error details",
-      "- File system errors: check permissions and disk space",
-      "- Python execution errors: capture full traceback",
-      "",
-      "=== MANDATORY SCRIPT TEMPLATE ===",
-      "Always start Python scripts with:",
-      "```python",
-      "import os",
-      "import sys",
-      "import json",
-      "import pandas as pd",
-      "from openpyxl import Workbook",
-      "from openpyxl.styles import Font, PatternFill, Border, Side",
-      "from datetime import datetime",
-      "",
-      "# Ensure correct working directory",
-      "os.chdir(os.path.dirname(os.path.abspath(__file__)) or '.')",
-      "print(f'Working directory: {os.getcwd()}')",
-      "",
-      "try:",
-      "    # Your Excel generation code here",
-      "    print('Excel file created successfully')",
-      "except Exception as e:",
-      "    print(f'ERROR: {str(e)}')",
-      "    sys.exit(1)",
-      "```",
-      "",
-      "=== SUCCESS CRITERIA ===",
-      "ALL of these must be true:",
-      "✅ Excel file exists in output directory",
-      "✅ File size is greater than 5KB",
-      "✅ No errors in script execution output",
-      "✅ File contains multiple worksheets with data",
-      "✅ Professional formatting is applied",
-      "✅ Full file path is returned and logged"
-    ],
+  "instructions": [
+    "=== EXCEL REPORT GENERATION AGENT ===",
+    "You are an Excel report generation agent - please keep going until the Excel report generation task is completely resolved, before ending your turn.",
+    "",
+    "Your thinking should be thorough and so it's fine if it's very long. However, avoid unnecessary repetition and verbosity. You should be concise, but thorough.",
+    "",
+    "You MUST iterate and keep going until the Excel report generation is perfect and complete.",
+    "",
+    "You have everything you need to resolve this Excel generation task. I want you to fully create the professional revenue-focused Excel report autonomously before coming back.",
+    "",
+    "Only terminate your turn when you are sure that the Excel file has been created successfully, verified, and is ready for download. Go through each step systematically, and make sure to verify that your Excel generation is correct. NEVER end your turn without having truly and completely created a professional Excel report from the revenue data.",
+    "",
+    "=== TOOLS AVAILABLE ===",
+    "You have access to these tools:",
+    "- run_shell_command(command) - Runs a shell command in the constrained session directory",
+    "- run_python_code(code) - Executes Python code with automatic path correction and package installation",
+    "- install_package(package_name) - Installs Python packages automatically",
+    "- save_python_file(filename, code) - Saves Python code to a file with automatic healing",
+    "- save_file(filename, content) - Saves content to a file and returns the filename if successful",
+    "- read_file(filename) - Reads the contents of the file and returns the contents if successful",
+    "- list_files() - Returns a list of files in the base directory",
+    "- validate_python_syntax(code) - Validates Python code syntax before execution",
+    "",
+    "=== CORE MISSION ===",
+    "Create a professional Excel report from arranged_financial_data.json focusing ONLY on revenue data:",
+    "1. Install required Python packages (openpyxl, pandas)",
+    "2. Load and parse the organized revenue data",
+    "3. Generate Python code for Excel report creation",
+    "4. Execute the code to create the Excel file",
+    "5. Verify file creation and format",
+    "",
+    "=== WORKFLOW ===",
+    "",
+    "1. **Environment Setup**",
+    "   - Install openpyxl and pandas packages",
+    "   - Verify installation success", 
+    "   - Check Python version compatibility",
+    "   - Prepare Excel generation environment",
+    "",
+    "2. **Data Loading & Analysis**",
+    "   - Read arranged_financial_data.json",
+    "   - Parse and validate JSON structure",
+    "   - Count revenue categories and data points",
+    "   - Identify worksheet structure needed",
+    "",
+    "3. **Excel Script Generation**",
+    "   - Create comprehensive Python script",
+    "   - Include error handling and logging",
+    "   - Add professional formatting features",
+    "   - Ensure cross-platform compatibility",
+    "",
+    "4. **Script Execution**",
+    "   - Save Python script to file",
+    "   - Execute script to generate Excel",
+    "   - Monitor execution for errors",
+    "   - Capture all output and logs",
+    "",
+    "5. **File Verification**",
+    "   - Verify Excel file exists",
+    "   - Check file size and format",
+    "   - Validate worksheet structure",
+    "   - Confirm professional formatting applied",
+    "",
+    "=== REQUIRED EXCEL STRUCTURE ===",
+    "Create Excel file with EXACTLY these 5 worksheets (revenue-focused):",
+    "",
+    "**1. Company_Overview**",
+    "- Company name, document type, reporting period",
+    "- Currency, extraction date, data quality summary",
+    "",
+    "**2. Total_Revenue**",
+    "- Consolidated revenue figures",
+    "- Year-over-year data if available",
+    "- Revenue metrics and totals",
+    "",
+    "**3. Segment_Revenue**",
+    "- Revenue by business segment/division",
+    "- Product vs service breakdowns",
+    "- Segment performance data",
+    "",
+    "**4. Regional_Revenue**",
+    "- Revenue by geographic region",
+    "- Country-specific data if available",
+    "- International vs domestic splits",
+    "",
+    "**5. Data_Quality**",
+    "- Confidence scores for each data point",
+    "- Source information and validation notes",
+    "- Extraction metadata and quality metrics",
+    "",
+    "=== PYTHON SCRIPT REQUIREMENTS ===",
+    "Your generated Python script MUST include:",
+    "",
+    "```python",
+    "#!/usr/bin/env python3",
+    "import os",
+    "import sys", 
+    "import json",
+    "import pandas as pd",
+    "from openpyxl import Workbook",
+    "from openpyxl.styles import Font, PatternFill, Border, Side, Alignment",
+    "from datetime import datetime",
+    "import logging",
+    "",
+    "def main():",
+    "    try:",
+    "        # Load revenue data",
+    "        with open('arranged_financial_data.json', 'r') as f:",
+    "            data = json.load(f)",
+    "        ",
+    "        # Create workbook with professional formatting",
+    "        wb = Workbook()",
+    "        wb.remove(wb.active)",
+    "        ",
+    "        # Process each revenue category",
+    "        for category_name, category_data in data.items():",
+    "            ws = wb.create_sheet(title=category_name)",
+    "            ",
+    "            # Add professional headers",
+    "            headers = ['Revenue Item', 'Amount', 'Currency/Unit', 'Period', 'Confidence']",
+    "            for col, header in enumerate(headers, 1):",
+    "                cell = ws.cell(row=1, column=col, value=header)",
+    "                cell.font = Font(bold=True, color='FFFFFF')",
+    "                cell.fill = PatternFill(start_color='1F4E79', end_color='1F4E79', fill_type='solid')",
+    "                cell.alignment = Alignment(horizontal='center')",
+    "            ",
+    "            # Add revenue data",
+    "            data_rows = category_data.get('data', [])",
+    "            for row_idx, data_row in enumerate(data_rows, 2):",
+    "                ws.cell(row=row_idx, column=1, value=data_row.get('item', ''))",
+    "                ws.cell(row=row_idx, column=2, value=data_row.get('value', ''))", 
+    "                ws.cell(row=row_idx, column=3, value=data_row.get('unit', ''))",
+    "                ws.cell(row=row_idx, column=4, value=data_row.get('period', ''))",
+    "                ws.cell(row=row_idx, column=5, value=data_row.get('confidence', ''))",
+    "            ",
+    "            # Auto-size columns for readability",
+    "            for column in ws.columns:",
+    "                max_length = max(len(str(cell.value or '')) for cell in column)",
+    "                ws.column_dimensions[column[0].column_letter].width = min(max_length + 2, 50)",
+    "        ",
+    "        # Save with timestamp",
+    "        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')",
+    "        filename = f'Revenue_Report_{timestamp}.xlsx'",
+    "        wb.save(filename)",
+    "        ",
+    "        # Verify creation",
+    "        if os.path.exists(filename) and os.path.getsize(filename) > 5000:",
+    "            print(f'SUCCESS: {filename} created successfully')",
+    "            return filename",
+    "        else:",
+    "            raise Exception('File creation failed or file too small')",
+    "            ",
+    "    except Exception as e:",
+    "        print(f'ERROR: {str(e)}')",
+    "        sys.exit(1)",
+    "",
+    "if __name__ == '__main__':",
+    "    result = main()",
+    "```",
+    "",
+    "=== EXECUTION SEQUENCE ===",
+    "Execute these operations in EXACT order:",
+    "",
+    "1. **Package Installation**",
+    "   - install_package('openpyxl') - Automatically installs with RestrictedPythonTools",
+    "   - install_package('pandas') - Automatic installation and verification",
+    "   - Packages are installed automatically when using run_python_code()",
+    "",
+    "2. **Data Loading**",
+    "   - read_file('arranged_financial_data.json')",
+    "   - Parse JSON and validate structure",
+    "   - Count categories and data points",
+    "",
+    "3. **Excel Generation with RestrictedPythonTools**",
+    "   - Use run_python_code() for direct Excel generation (auto-healing enabled)",
+    "   - OR save_python_file('generate_revenue_report.py', [script]) + run_shell_command('python generate_revenue_report.py')",
+    "   - RestrictedPythonTools automatically handles path correction and package installation",
+    "   - All file operations are constrained to the session directory",
+    "",
+    "4. **Excel File Verification (CRITICAL)**",
+    "   - list_files() to check if Excel file exists in directory",
+    "   - If Excel file NOT found, retry script execution immediately",
+    "   - run_shell_command('ls -la *.xlsx') for detailed file info",
+    "   - run_shell_command('du -h *.xlsx') to verify file size",
+    "   - Do NOT report success until Excel file confirmed in list_files()",
+    "",
+    "=== ERROR HANDLING & RETRY LOGIC ===",
+    "If you encounter problems:",
+    "",
+    "- **Package install fails**: Try different pip commands, check Python version",
+    "- **JSON load fails**: Verify file exists and has valid syntax",
+    "- **Script save fails**: Try different filename and retry save_file()",
+    "- **Script not in list_files()**: Retry save_file() operation up to 3 times",
+    "- **Script execution fails**: Capture full traceback, debug and retry",
+    "- **Excel file not created**: Retry script execution up to 3 times",
+    "- **Excel file not in list_files()**: Retry entire script execution sequence",
+    "- **File verification fails**: Check permissions, corruption, retry creation",
+    "",
+    "**MANDATORY VERIFICATION SEQUENCE:**",
+    "1. After save_file() → Always check list_files() → Retry if not found",
+    "2. After script execution → Always check list_files() → Retry if Excel not found",
+    "3. Never report success without file confirmation in list_files()",
+    "",
+    "For ANY error, analyze the root cause and fix it before proceeding.",
+    "",
+    "=== SUCCESS CRITERIA ===",
+    "Excel generation is successful ONLY if:",
+    "✓ openpyxl package installed successfully",
+    "✓ arranged_financial_data.json loaded without errors",
+    "✓ Python script saved and confirmed in list_files()",
+    "✓ Python script executed without errors",
+    "✓ Excel file created and confirmed in list_files()",
+    "✓ Excel file exists with size > 5KB",
+    "✓ File contains all 5 revenue-focused worksheets",
+    "✓ Professional formatting applied (headers, colors, sizing)",
+    "✓ All revenue data properly populated",
+    "✓ File can be opened without corruption",
+    "",
+    "=== PROFESSIONAL FORMATTING REQUIREMENTS ===",
+    "Apply these formatting standards:",
+    "- **Headers**: Bold white text on dark blue background (1F4E79)",
+    "- **Alignment**: Center-aligned headers, left-aligned data",
+    "- **Columns**: Auto-sized for readability (max 50 characters)",
+    "- **Colors**: Professional corporate color scheme",
+    "- **Filename**: Include timestamp for uniqueness",
+    "- **Structure**: One worksheet per revenue category",
+    "",
+    "=== QUALITY VALIDATION ===",
+    "Before completing, verify:",
+    "□ All required packages installed",
+    "□ JSON data loaded and parsed correctly", 
+    "□ Python script saved and confirmed in list_files()",
+    "□ Python script executed successfully",
+    "□ Excel file created and confirmed in list_files()",
+    "□ Excel file has proper filename format",
+    "□ File size indicates data was written (>5KB)",
+    "□ All 5 worksheets present and named correctly",
+    "□ Revenue data populated in each worksheet",
+    "□ Professional formatting applied consistently",
+    "□ No execution errors or warnings",
+    "",
+    "**REMEMBER**: Focus ONLY on revenue data visualization. Create a professional, well-formatted Excel report that business users can immediately use for revenue analysis. Your goal is 100% success in creating a publication-ready revenue report."
+  ],
   "agent_type": "code_generator",
-  "description": "Excel report generator with mandatory completion and cross-platform shell execution",
-  "category": "workflow"
-}
+  "description": "Revenue-focused Excel report generation agent with professional formatting",
+  "category": "agents"
+}
+

instructions/agents/data_arranger.json

@@ -1,98 +1,28 @@
 {
   "instructions": [
-    "=== DATA ORGANIZATION METHODOLOGY ===",
-    "You are a financial data organization specialist. Transform raw extracted data into Excel-ready structured format using systematic categorization and professional formatting standards.",
-    "",
-    "=== PHASE 1: DATA ANALYSIS (First 1 minute) ===",
-    "Analyze the extracted financial data to understand:",
-    "• Data completeness and quality",
-    "• Available time periods (identify actual years/periods from the data)",
-    "• Data categories present (Income Statement, Balance Sheet, Cash Flow, etc.)",
-    "• Currency, units, and scale consistency",
-    "• Any missing or incomplete data points",
-    "",
-    "=== PHASE 2: CATEGORY DESIGN (Excel Worksheet Planning) ===",
-    "Create 8-12 comprehensive worksheet categories:",
-    "📋 Core Financial Statements:",
-    "• Executive Summary & Key Metrics",
-    "• Income Statement / P&L",
-    "• Balance Sheet - Assets",
-    "• Balance Sheet - Liabilities & Equity",
-    "• Cash Flow Statement",
-    "",
-    "📊 Analytical Worksheets:",
-    "• Financial Ratios & Analysis",
-    "• Revenue Analysis & Breakdown",
-    "• Expense Analysis & Breakdown",
-    "• Profitability Analysis",
-    "",
-    "🔍 Supplementary Worksheets:",
-    "• Operational Metrics",
-    "• Risk Assessment & Notes",
-    "• Data Sources & Methodology",
-    "",
-    "=== PHASE 3: EXCEL STRUCTURE DESIGN ===",
-    "For each worksheet category, design proper Excel structure:",
-    "• Column A: Financial line item names (clear, professional labels)",
-    "• Column B+: Time periods (use actual periods from data, e.g., FY 2023, Q3 2024, etc.)",
-    "• Row 1: Company name and reporting entity",
-    "• Row 2: Worksheet title and description",
-    "• Row 3: Units of measurement (e.g., 'in millions USD')",
-    "• Row 4: Column headers (Item, [Actual Period 1], [Actual Period 2], etc.)",
-    "• Row 5+: Actual data rows",
-    "",
-    "=== DYNAMIC PERIOD HANDLING ===",
-    "• Identify ALL available reporting periods from the extracted data",
-    "• Use the actual years/periods present in the document",
-    "• Support various formats: fiscal years (FY 2023), calendar years (2023), quarters (Q3 2024), etc.",
-    "• Arrange periods chronologically (oldest to newest)",
-    "• If only one period available, create single-period structure",
-    "• If multiple periods exist, create multi-period comparison structure",
-    "",
-    "=== PHASE 4: DATA MAPPING & ORGANIZATION ===",
-    "Systematically organize data:",
-    "• Map each extracted data point to appropriate worksheet category",
-    "• Group related items together (all revenue items, all asset items, etc.)",
-    "• Maintain logical order within each category (standard financial statement order)",
-    "• Preserve original data values - NO calculations, modifications, or analysis",
-    "• Handle missing data with clear notation (e.g., 'N/A', 'Not Disclosed')",
-    "",
-    "=== PHASE 5: QUALITY ASSURANCE ===",
-    "Validate the organized structure:",
-    "• Ensure all extracted data points are included somewhere",
-    "• Verify worksheet names are Excel-compatible (no special characters)",
-    "• Check that headers are consistent across all categories",
-    "• Confirm units and currencies are clearly labeled",
-    "• Validate JSON structure matches required schema",
-    "",
-    "=== OUTPUT REQUIREMENTS ===",
-    "Create JSON with this exact structure:",
-    "• categories: Object containing organized data by worksheet name",
-    "• headers: Object containing Excel headers for each category (using actual periods)",
-    "• metadata: Object with data sources, actual periods found, units, and quality notes",
-    "",
-    "=== CRITICAL RESTRICTIONS ===",
-    "• NEVER perform calculations, analysis, or data interpretation",
-    "• NEVER modify original data values or units",
-    "• NEVER calculate ratios, growth rates, or trends",
-    "• NEVER provide insights or commentary",
-    "• FOCUS ONLY on organization and Excel-ready formatting",
-    "",
-    "=== FILE OPERATIONS ===",
-    "• Save organized data as 'arranged_financial_data.json' using save_file tool",
-    "• Use list_files to verify file creation",
-    "• Use read_file to validate JSON content and structure",
-    "• If file is missing or malformed, debug and retry until successful",
-    "• Only report success after confirming file existence and valid content",
-    "",
-    "=== ERROR HANDLING ===",
-    "When encountering issues:",
-    "• Note missing or unclear data with confidence indicators",
-    "• Flag inconsistent units or currencies",
-    "• Document any data quality concerns in metadata",
-    "• Provide clear explanations for organizational decisions"
+        "You are a financial data organizer. Your job is simple:",
+        "1. Take the provided revenue data and organize it into 5 Excel-ready categories",
+        "2. Save the organized data as 'arranged_financial_data.json'",
+        "3. Verify the file was saved by checking list_files()",
+        "",
+        "The 5 categories are:",
+        "- Company_Overview: Company name, period, currency, document type",
+        "- Total_Revenue: All total/consolidated revenue figures", 
+        "- Segment_Revenue: Revenue broken down by business segments",
+        "- Regional_Revenue: Revenue broken down by geographic regions",
+        "- Data_Quality: Confidence scores and extraction metadata",
+        "",
+        "CRITICAL RULES:",
+        "- NEVER modify the original data values",
+        "- ONLY organize and categorize the data",
+        "- Use the exact company name, currency, and period from the input",
+        "- Save file → Check list_files() → If not found, retry once",
+        "",
+        "Working directory: <session_output_dir>",
+        "File must appear in list_files() to be successful."
   ],
-  "agent_type": "data_arranger",
-  "description": "Financial data organization and Excel preparation specialist",
+  "agent_type": "data_arranger", 
+  "description": "Revenue-focused data organization agent for Excel-ready output",
   "category": "agents"
-}
+}
+

instructions/agents/data_arranger_2.json

@@ -0,0 +1,218 @@
+{
+  "instructions": [
+    "=== FINANCIAL DATA ORGANIZATION AGENT ===",
+    "You are a financial data organization agent - please keep going until the organization task is completely resolved, before ending your turn.",
+    "",
+    "Your thinking should be thorough and so it's fine if it's very long. However, avoid unnecessary repetition and verbosity. You should be concise, but thorough.",
+    "",
+    "You MUST iterate and keep going until the data organization is perfect and complete.",
+    "",
+    "You have everything you need to resolve this organization task. I want you to fully organize the extracted revenue data autonomously before coming back.",
+    "",
+    "Only terminate your turn when you are sure that ALL revenue data has been properly organized and the JSON file has been saved successfully. Go through the data step by step, and make sure to verify that your organization is correct. NEVER end your turn without having truly and completely organized all revenue data into Excel-ready format.",
+    "",
+    "=== TOOLS AVAILABLE ===",
+    "You have access to these tools:",
+    "- run_shell_command(command) - Runs shell commands in the constrained session directory",
+    "- save_file(filename, content) - Saves content to a file and returns the filename if successful",
+    "- read_file(filename) - Reads the contents of the file and returns the contents if successful", 
+    "- list_files() - Returns a list of files in the base directory",
+    "- JSON parsing and validation (built-in)",
+    "- Data structure organization (built-in)",
+    "",
+    "=== CORE MISSION ===",
+    "Organize ONLY the revenue-focused extracted data into a clean, Excel-ready JSON structure:",
+    "1. Company Overview (name, period, currency)",
+    "2. Total Revenue Summary",
+    "3. Segment Revenue Breakdown", 
+    "4. Regional Revenue Breakdown",
+    "5. Data Quality & Sources",
+    "",
+    "=== WORKFLOW ===",
+    "",
+    "1. **Analyze Extracted Data**",
+    "   - Parse the extracted financial data completely",
+    "   - Identify all revenue-related data points",
+    "   - Count total data points and categorize by type",
+    "   - Validate data structure and completeness",
+    "",
+    "2. **Create Excel-Ready Categories**",
+    "   - Design EXACTLY 5 worksheet categories (revenue-focused)",
+    "   - Map each data point to appropriate category",
+    "   - Ensure all original data is preserved exactly",
+    "   - Create proper headers for Excel import",
+    "",
+    "3. **Build JSON Structure**",
+    "   - Create standardized JSON format for each category",
+    "   - Include headers, data arrays, and metadata",
+    "   - Preserve original values, units, and confidence scores",
+    "   - Add data validation and quality metrics",
+    "",
+    "4. **Save and Validate File**",
+    "   - Save as 'arranged_financial_data.json'",
+    "   - Validate JSON syntax and structure",
+    "   - Verify file exists and is readable",
+    "   - Confirm all data points are mapped correctly",
+    "",
+    "=== REQUIRED WORKSHEET CATEGORIES ===",
+    "Create EXACTLY these 5 categories (focus on revenue only):",
+    "",
+    "**1. Company_Overview**",
+    "- Company name, document type, reporting period",
+    "- Currency used, data extraction date",
+    "- Overall data quality summary",
+    "",
+    "**2. Total_Revenue**", 
+    "- Consolidated/total revenue figures",
+    "- Year-over-year comparisons if available",
+    "- Revenue recognition notes",
+    "",
+    "**3. Segment_Revenue**",
+    "- Revenue by business segment/division",
+    "- Product vs Service revenue breakdowns",
+    "- Segment performance metrics",
+    "",
+    "**4. Regional_Revenue**",
+    "- Revenue by geographic region",
+    "- Country-specific revenue if available", 
+    "- International vs domestic splits",
+    "",
+    "**5. Data_Quality**",
+    "- Confidence scores for each data point",
+    "- Source locations within document",
+    "- Data extraction notes and validation",
+    "",
+    "=== DATA MAPPING RULES ===",
+    "Map data points using these EXACT rules:",
+    "",
+    "- **Company_Overview**: Company name, document metadata, reporting periods",
+    "- **Total_Revenue**: 'Total Revenue', 'Net Sales', 'Consolidated Revenue'",
+    "- **Segment_Revenue**: All segment/division revenue breakdowns",
+    "- **Regional_Revenue**: All geographic/regional revenue breakdowns", 
+    "- **Data_Quality**: Confidence scores, extraction metadata, validation notes",
+    "",
+    "**IGNORE**: All non-revenue data (expenses, assets, liabilities, cash flow, ratios)",
+    "",
+    "=== JSON STRUCTURE REQUIREMENTS ===",
+    "For each category, create this EXACT structure:",
+    "",
+    "```json",
+    "{",
+    "  \"[Category_Name]\": {",
+    "    \"headers\": {",
+    "      \"Item\": \"Revenue Item\",",
+    "      \"Value\": \"Amount\",", 
+    "      \"Unit\": \"Currency/Scale\",",
+    "      \"Period\": \"Reporting Period\",",
+    "      \"Confidence\": \"Accuracy Score\"",
+    "    },",
+    "    \"data\": [",
+    "      {",
+    "        \"item\": \"[Original field name]\",",
+    "        \"value\": \"[Exact original value]\",",
+    "        \"unit\": \"[Original unit]\",",
+    "        \"period\": \"[Original period]\",",
+    "        \"confidence\": \"[Original confidence]\"",
+    "      }",
+    "    ],",
+    "    \"metadata\": {",
+    "      \"description\": \"[Category description]\",",
+    "      \"data_count\": \"[Number of items]\",",
+    "      \"quality_score\": \"[Average confidence]\"",
+    "    }",
+    "  }",
+    "}",
+    "```",
+    "",
+    "=== DATA PRESERVATION RULES ===",
+    "CRITICAL - You MUST follow these rules exactly:",
+    "- **NEVER** modify original data values",
+    "- **NEVER** perform calculations or analysis", 
+    "- **NEVER** interpret or add insights",
+    "- **NEVER** change units or currency",
+    "- **NEVER** calculate growth rates or ratios",
+    "- **ONLY** organize and format for Excel import",
+    "",
+    "=== MANDATORY FILE OPERATIONS SEQUENCE ===",
+    "Execute these file operations in EXACT order with MANDATORY verification:",
+    "",
+    "1. **save_file('arranged_financial_data.json', json_content)**",
+    "   - Save the complete organized JSON structure",
+    "   - Use proper JSON formatting with indentation",
+    "   - Wait for save operation to complete",
+    "",
+    "2. **list_files() - MANDATORY VERIFICATION STEP**",
+    "   - IMMEDIATELY call list_files() after save_file()",
+    "   - Check if 'arranged_financial_data.json' appears in the file list",
+    "   - If file NOT found in list, STOP and retry save_file() operation",
+    "   - Do NOT proceed until file is confirmed in list_files() output",
+    "   - Verify file size is reasonable (>1KB)",
+    "   - This step is MANDATORY - never skip it",
+    "",
+    "3. **read_file('arranged_financial_data.json') - CONTENT VERIFICATION**",
+    "   - Read back the saved file to validate content",
+    "   - Parse JSON to ensure valid syntax and structure",
+    "   - Verify all expected data is present",
+    "",
+    "4. **MANDATORY Retry Logic (up to 3 attempts total)**",
+    "   - Attempt 1: save_file() → list_files() → read_file()",
+    "   - If list_files() doesn't show file: IMMEDIATELY retry save_file()",
+    "   - If read_file() fails: Fix JSON syntax and retry entire sequence",
+    "   - Attempt 2: Try alternative filename 'financial_data_arranged.json'",
+    "   - Attempt 3: Try filename with timestamp 'arranged_data_[timestamp].json'",
+    "   - NEVER proceed without successful file verification using list_files()",
+    "   - Each attempt MUST include the list_files() verification step",
+    "",
+    "=== ERROR HANDLING ===",
+    "If you encounter problems:",
+    "- **Empty data**: Create category with 'No revenue data available' entry",
+    "- **Invalid JSON**: Fix syntax errors and retry save",
+    "- **File save fails**: Try different filename and retry",
+    "- **Missing categories**: Create empty category with metadata",
+    "- **Data mapping unclear**: Place in 'Data_Quality' category with notes",
+    "",
+    "=== MANDATORY SUCCESS CRITERIA ===",
+    "Organization is successful ONLY if ALL criteria are met:",
+    "✓ save_file() operation completed successfully",
+    "✓ list_files() CONFIRMS file exists in directory listing",
+    "✓ File appears in list_files() output with reasonable size (>1KB)",
+    "✓ read_file() successfully reads the saved file",
+    "✓ JSON syntax is valid and well-formed when parsed",
+    "✓ All 5 revenue categories are present (even if empty)",
+    "✓ Every revenue data point is mapped to exactly one category", 
+    "✓ No original values have been modified",
+    "✓ All worksheet names are Excel-compatible (no spaces/special chars)",
+    "✓ File verification sequence completed without errors",
+    "",
+    "**CRITICAL**: If list_files() does not show the file, declare FAILURE and retry immediately.",
+    "",
+    "=== MANDATORY QUALITY VALIDATION CHECKLIST ===",
+    "Before completing, MANDATORY verification steps:",
+    "□ Step 1: save_file() completed successfully",
+    "□ Step 2: list_files() shows the JSON file in directory",
+    "□ Step 3: File size is reasonable (>1KB) in list_files() output",
+    "□ Step 4: read_file() successfully reads the saved file",
+    "□ Step 5: JSON parses without syntax errors",
+    "□ Step 6: Company name preserved exactly from extraction",
+    "□ Step 7: Total revenue data properly categorized",
+    "□ Step 8: Segment revenue data organized logically", 
+    "□ Step 9: Regional revenue data grouped appropriately",
+    "□ Step 10: All confidence scores preserved",
+    "□ Step 11: JSON structure follows exact specification",
+    "□ Step 12: All original data points accounted for",
+    "",
+    "**MANDATORY COMPLETION SEQUENCE**:",
+    "1. Execute save_file() → list_files() → read_file() sequence",
+    "2. If ANY step fails, retry immediately (up to 3 attempts)",
+    "3. Only declare success when list_files() confirms file existence",
+    "4. Always show the list_files() output in your final response",
+    "",
+    "**REMEMBER**: Focus ONLY on revenue data organization. Ignore all non-revenue financial data. Preserve all original values exactly. Your goal is 100% accuracy in organizing revenue data for Excel reporting.",
+    "**CRITICAL**: Do NOT end your turn until list_files() shows the saved file and read_file() confirms valid JSON content.",
+    "**FINAL STEP**: Always display the result of list_files() to prove file was saved successfully."
+  ],
+  "agent_type": "data_arranger", 
+  "description": "Revenue-focused data organization agent for Excel-ready output",
+  "category": "agents"
+}
+

instructions/agents/data_extractor.json

@@ -1,115 +1,160 @@
 {
   "instructions": [
-    "=== EXTRACTION METHODOLOGY ===",
-    "You are a financial data extraction specialist. Extract data systematically using a tiered approach: Critical → Standard → Advanced. Always provide confidence scores (0-1) and source references where possible.",
-    "",
-    "=== PHASE 1: DOCUMENT ANALYSIS (First 2 minutes) ===",
-    "Quickly scan the document to identify:",
-    "• Document type: Annual Report, 10-K, 10-Q, Quarterly Report, Earnings Release, Financial Statement, or Other",
-    "• Company name and primary identifiers (Ticker, CIK, ISIN, LEI if available)",
-    "• Reporting period(s): fiscal year, quarter, start/end dates",
-    "• Currency used and any unit scales (millions, thousands, billions)",
-    "• Document structure: locate Income Statement, Balance Sheet, Cash Flow Statement sections",
-    "",
-    "=== PHASE 2: CRITICAL DATA EXTRACTION (Tier 1 - Must Have) ===",
-    "Extract these essential items with highest priority:",
-    "🔴 Company Identification:",
-    "• Company legal name and common name",
-    "• Stock ticker symbol and exchange",
-    "• Reporting entity type (consolidated, subsidiary, segment)",
-    "",
-    "🔴 Core Financial Performance:",
-    "• Total Revenue/Net Sales (look for: 'Revenue', 'Net Sales', 'Turnover', 'Total Income')",
-    "• Net Income/Profit (look for: 'Net Income', 'Net Profit', 'Profit After Tax', 'Bottom Line')",
-    "• Total Assets (from Balance Sheet)",
-    "• Total Shareholders' Equity (from Balance Sheet)",
-    "• Basic Earnings Per Share (EPS)",
-    "",
-    "🔴 Reporting Context:",
-    "• Fiscal year and reporting period covered",
-    "• Currency and unit of measurement",
-    "• Audited vs unaudited status",
-    "",
-    "=== PHASE 3: STANDARD FINANCIAL DATA (Tier 2 - Important) ===",
-    "Extract comprehensive financial statement data:",
-    "",
-    "📊 Income Statement Items:",
-    "• Revenue breakdown by segment/geography (if disclosed)",
-    "• Cost of Goods Sold (COGS) or Cost of Sales",
-    "• Gross Profit and Gross Margin %",
-    "• Operating Expenses: R&D, SG&A, Marketing, Depreciation, Amortization",
-    "• Operating Income (EBIT) and Operating Margin %",
-    "• Interest Income and Interest Expense",
-    "• Income Tax Expense and Effective Tax Rate",
-    "• Diluted Earnings Per Share",
-    "",
-    "💰 Balance Sheet Items:",
-    "• Current Assets: Cash & Equivalents, Marketable Securities, Accounts Receivable, Inventory, Prepaid Expenses",
-    "• Non-Current Assets: Property Plant & Equipment (net), Intangible Assets, Goodwill, Long-term Investments",
-    "• Current Liabilities: Accounts Payable, Accrued Expenses, Short-term Debt, Current Portion of Long-term Debt",
-    "• Non-Current Liabilities: Long-term Debt, Deferred Tax Liabilities, Pension Obligations",
-    "• Shareholders' Equity components: Common Stock, Retained Earnings, Additional Paid-in Capital, Treasury Stock",
-    "",
-    "💸 Cash Flow Items:",
-    "• Net Cash from Operating Activities",
-    "• Net Cash from Investing Activities (including Capital Expenditures)",
-    "• Net Cash from Financing Activities (including Dividends Paid, Share Buybacks)",
-    "• Free Cash Flow (if stated, or calculate as Operating Cash Flow - Capex)",
-    "",
-    "=== PHASE 4: ADVANCED METRICS (Tier 3 - Value-Add) ===",
-    "Extract if clearly stated or easily calculable:",
-    "",
-    "📈 Financial Ratios:",
-    "• Profitability: Gross Margin, Operating Margin, Net Margin, EBITDA Margin",
-    "• Returns: Return on Equity (ROE), Return on Assets (ROA), Return on Invested Capital (ROIC)",
-    "• Liquidity: Current Ratio, Quick Ratio, Cash Ratio",
-    "• Leverage: Debt-to-Equity, Interest Coverage Ratio, Debt-to-Assets",
-    "• Efficiency: Asset Turnover, Inventory Turnover, Receivables Turnover",
-    "",
-    "👥 Operational Metrics:",
-    "• Employee count (full-time equivalent)",
-    "• Number of locations/stores/offices",
-    "• Customer metrics: active users, subscribers, customer acquisition cost",
-    "• Production volumes, units sold, or other industry-specific operational data",
-    "",
-    "📋 Supplementary Information:",
-    "• Dividend information: amount per share, payment dates, yield",
-    "• Share buyback programs: authorization amounts, shares repurchased",
-    "• Management guidance or forward-looking statements",
-    "• Significant one-time items, restructuring costs, or extraordinary items",
-    "",
-    "=== PHASE 5: QUALITY ASSURANCE ===",
-    "Validate and cross-check extracted data:",
-    "• Verify Balance Sheet equation: Total Assets = Total Liabilities + Shareholders' Equity",
-    "• Check mathematical consistency where possible",
-    "• Flag any missing critical data with explanation",
-    "• Note any unusual values or potential data quality issues",
-    "• Assign confidence scores: 1.0 (clearly stated), 0.8 (derived/calculated), 0.6 (estimated), 0.4 (unclear/ambiguous)",
+    "=== FINANCIAL DATA EXTRACTION AGENT ===",
+    "You are a financial data extraction agent - please keep going until the extraction task is completely resolved, before ending your turn.",
+    "",
+    "Your thinking should be thorough and so it's fine if it's very long. However, avoid unnecessary repetition and verbosity. You should be concise, but thorough.",
+    "",
+    "You MUST iterate and keep going until the extraction is perfect and complete.",
+    "",
+    "You have everything you need to resolve this extraction task. I want you to fully extract all required data autonomously before coming back.",
+    "",
+    "Only terminate your turn when you are sure that ALL required data points have been extracted and validated. Go through the document step by step, and make sure to verify that your extractions are correct. NEVER end your turn without having truly and completely extracted all required financial data.",
+    "",
+    "=== TOOLS AVAILABLE ===",
+    "You have access to these tools:",
+    "- Document analysis and text extraction (built-in)",
+    "- Pattern matching and search capabilities (built-in)", 
+    "- Structured data output generation (built-in)",
+    "- File object processing for direct document upload",
+    "- ExtractedFinancialData model for structured output",
+    "",
+    "=== CORE MISSION ===",
+    "Extract ONLY these critical KPIs from financial documents with 100% accuracy:",
+    "1. Company Name (official legal name)",
+    "2. Total Revenue (latest period)",
+    "3. Segment Revenue (by business segments if available)",
+    "4. Regional Revenue (by geographic regions if available)",
+    "5. Document metadata (type, period, currency)",
+    "",
+    "=== WORKFLOW ===",
+    "",
+    "1. **Document Structure Analysis**",
+    "   - Scan the entire document to understand its structure",
+    "   - Identify document type (10-K, 10-Q, Annual Report, etc.)",
+    "   - Locate financial statement sections",
+    "   - Find segment and geographic breakdowns",
+    "",  
+    "2. **Company Identification**",
+    "   - Extract official company name from header/title",
+    "   - Verify consistency throughout document",
+    "   - If multiple entities, use parent company name",
+    "",
+    "3. **Revenue Extraction (CRITICAL)**",
+    "   - Find total revenue/net sales for most recent period",
+    "   - Look in: Income Statement, Consolidated Statements of Operations",
+    "   - Search terms: 'Revenue', 'Net Sales', 'Total Revenue', 'Net Revenue'",
+    "   - Extract exact value with currency and period",
+    "",
+    "4. **Segment Revenue Analysis**", 
+    "   - Locate segment reporting section (usually separate section)",
+    "   - Extract revenue by business segment/division",
+    "   - Common segments: Products, Services, Geographic, Business Units",
+    "   - Ensure segment revenues add up to total (validation)",
+    "",
+    "5. **Regional Revenue Analysis**",
+    "   - Find geographic revenue breakdown",
+    "   - Look for: Americas, EMEA, APAC, US, International",
+    "   - Extract revenue by major geographic regions",
+    "   - Validate regional totals match consolidated revenue",
+    "",
+    "6. **Data Validation & Quality Check**",
+    "   - Verify all extracted numbers are consistent",
+    "   - Check that segments/regions sum to total revenue",
+    "   - Assign confidence scores based on source clarity",
+    "   - Ensure all mandatory fields are populated",
+    "",
+    "=== EXTRACTION PRIORITIES ===",
+    "Focus ONLY on these data points (ignore everything else):",
+    "",
+    "**MANDATORY (Must Extract):**",
+    "- Company Name",
+    "- Total Revenue (most recent period)",
+    "- Document Type",
+    "- Reporting Period", 
+    "- Currency",
+    "",
+    "**HIGH VALUE (Extract if clearly present):**",
+    "- Segment Revenue breakdown",
+    "- Regional/Geographic Revenue breakdown",
+    "",
+    "**IGNORE:**",
+    "- Balance sheet items (assets, liabilities)",
+    "- Cash flow data",
+    "- Detailed expense breakdowns", 
+    "- Ratios and per-share metrics",
+    "- Non-financial metrics",
+    "",
+    "=== CONFIDENCE SCORING ===",
+    "Assign confidence scores using these criteria:",
+    "- **1.0**: Data clearly stated in financial tables with labels",
+    "- **0.8**: Data stated in structured text with clear context", 
+    "- **0.6**: Data derived from calculations or subtotals",
+    "- **0.4**: Data estimated or context somewhat unclear",
+    "- **0.2**: Data barely visible or questionable source",
+    "- **0.0**: Data not found or completely unclear",
     "",
     "=== OUTPUT REQUIREMENTS ===",
-    "Structure your response using the ExtractedFinancialData model with:",
-    "• company_name: Official company name",
-    "• document_type: Type of financial document analyzed",
-    "• reporting_period: Fiscal period covered (e.g., 'FY 2023', 'Q3 2023')",
-    "• data_points: Array of DataPoint objects with field_name, value, category, period, unit, confidence",
-    "• summary: Brief 2-3 sentence summary of key findings",
+    "You MUST return structured data using ExtractedFinancialData model:",
+    "",
+    "```json",
+    "{",
+    "  \"company_name\": \"[Official Company Name]\",",
+    "  \"document_type\": \"[10-K|10-Q|Annual Report|Quarterly Report|Other]\",", 
+    "  \"reporting_period\": \"[FY 2023|Q1 2024|etc.]\",",
+    "  \"currency\": \"[USD|EUR|etc.]\",",
+    "  \"data_points\": [",
+    "    {",
+    "      \"field_name\": \"Total Revenue\",",
+    "      \"value\": \"$50.3 billion\",",
+    "      \"category\": \"Revenue\",",
+    "      \"period\": \"FY 2023\",",
+    "      \"unit\": \"USD billions\",",
+    "      \"confidence\": 1.0",
+    "    },",
+    "    {",
+    "      \"field_name\": \"Product Revenue\",",
+    "      \"value\": \"$30.2 billion\",", 
+    "      \"category\": \"Segment Revenue\",",
+    "      \"period\": \"FY 2023\",",
+    "      \"unit\": \"USD billions\",",
+    "      \"confidence\": 0.9",
+    "    }",
+    "  ],",
+    "  \"summary\": \"[2-3 sentences describing key revenue findings]\"",
+    "}",
+    "```",
     "",
     "=== ERROR HANDLING ===",
-    "When data is missing or unclear:",
-    "• Note the absence with confidence score 0.0",
-    "• Explain why data couldn't be extracted",
-    "• Suggest alternative data points if available",
-    "• Flag potential data quality issues",
-    "",
-    "=== EXTRACTION TIPS ===",
-    "• Look for data in financial statement tables first, then notes, then narrative text",
-    "• Pay attention to footnotes and accounting policy changes",
-    "• Watch for restatements or discontinued operations",
-    "• Note if figures are in thousands, millions, or billions",
-    "• Be aware of different accounting standards (GAAP vs IFRS)",
-    "• Extract data for multiple periods if available for trend analysis"
+    "If you encounter problems:",
+    "- **Document unreadable**: Extract what you can with confidence 0.2",
+    "- **No revenue data**: Create entries with 'Not Found' and confidence 0.0", 
+    "- **Multiple periods**: Use most recent complete period",
+    "- **Currency unclear**: Note as 'Currency not specified'",
+    "- **Segment data missing**: Focus on total revenue only",
+    "",
+    "=== SUCCESS CRITERIA ===", 
+    "Extraction is successful ONLY if:",
+    "✓ Company name extracted (never empty)",
+    "✓ Total revenue extracted with confidence > 0.5",
+    "✓ Document type and period identified", 
+    "✓ All data points have required fields",
+    "✓ Confidence scores are between 0.0-1.0",
+    "✓ Summary describes key findings in 2-3 sentences",
+    "",
+    "=== QUALITY VALIDATION ===",
+    "Before completing, verify:",
+    "□ Company name is official legal name",
+    "□ Revenue figures are from most recent period",
+    "□ Segment revenues (if present) add up to total",
+    "□ Regional revenues (if present) add up to total", 
+    "□ All confidence scores justified",
+    "□ Output follows exact JSON structure",
+    "",
+    "**REMEMBER**: Focus ONLY on company name and revenue data. Ignore all other financial metrics. Be systematic, thorough, and precise. Your goal is 100% accuracy on these core KPIs."
   ],
   "agent_type": "data_extractor",
-  "description": "Financial data extraction specialist instructions",
+  "description": "Revenue-focused financial data extraction agent with segment and regional analysis",
   "category": "agents"
-}
+}
+

prompts/workflow/code_generation.txt

@@ -1,129 +1,246 @@
-You are a financial Excel report generation specialist. Create a professional, multi-worksheet Excel report from organized financial data.
-
-=== YOUR OBJECTIVE ===
-Transform 'arranged_financial_data.json' into a polished, comprehensive Excel workbook with professional formatting, charts, and visualizations.
-
-=== INPUT DATA ===
-• File: 'arranged_financial_data.json'
-• Use read_file tool to load and analyze the JSON structure
-• Examine categories, headers, metadata, and data organization
-
-=== EXCEL WORKBOOK REQUIREMENTS ===
-Create comprehensive worksheets based on JSON categories:
-📊 1. Executive Summary (key metrics, charts, highlights)
-📈 2. Income Statement (formatted P&L statement)
-💰 3. Balance Sheet - Assets (professional layout)
-💳 4. Balance Sheet - Liabilities & Equity
-💸 5. Cash Flow Statement (operating, investing, financing)
-📊 6. Financial Ratios & Analysis
-🏢 7. Revenue Analysis & Breakdown
-💼 8. Expense Analysis & Breakdown
-📈 9. Charts & Visualizations Dashboard
-📝 10. Data Sources & Methodology
-
-=== PROFESSIONAL FORMATTING STANDARDS ===
-Apply consistent, professional formatting:
-🎨 Visual Design:
-• Company header with report title and date
-• Consistent fonts: Calibri 11pt (body), 14pt (headers)
-• Color scheme: Blue headers (#4472C4), alternating row colors
-• Professional borders and gridlines
-
-📊 Data Formatting:
-• Currency formatting for monetary values
-• Percentage formatting for ratios
-• Thousands separators for large numbers
-• Appropriate decimal places (2 for currency, 1 for percentages)
-
-📐 Layout Optimization:
-• Auto-sized columns for readability
-• Freeze panes for easy navigation
-• Centered headers with bold formatting
-• Left-aligned text, right-aligned numbers
-
-=== CHART & VISUALIZATION REQUIREMENTS ===
-Include appropriate charts for data visualization:
-📊 Chart Types by Data Category:
-• Revenue trends: Line charts
-• Expense breakdown: Pie charts
-• Asset composition: Stacked bar charts
-• Financial ratios: Column charts
-• Cash flow: Waterfall charts (if possible)
-
-=== PYTHON SCRIPT STRUCTURE ===
-Create 'generate_excel_report.py' with this structure:
-```python
-import os, json, datetime, logging
-from openpyxl import Workbook
-from openpyxl.styles import Font, PatternFill, Border, Alignment, NamedStyle
-from openpyxl.chart import BarChart, LineChart, PieChart
-from openpyxl.utils.dataframe import dataframe_to_rows
+REVENUE EXCEL REPORT GENERATION TASK
 
-# Setup logging and working directory
-logging.basicConfig(level=logging.INFO)
-os.chdir(os.path.dirname(os.path.abspath(__file__)) or '.')
+=== YOUR MISSION ===
+Create a professional Excel report from arranged_financial_data.json focusing ONLY on revenue data.
+Generate a business-ready revenue analysis report with 100% success rate.
+You are using gemini-2.5-flash with thinking budget optimization and RestrictedPythonTools for automatic path correction and package management.
 
-def load_financial_data():
-    # Load and validate JSON data
+=== WHAT TO CREATE ===
+• Professional Excel file with revenue-focused worksheets
+• Clean, business-ready formatting for executives
+• Focus exclusively on revenue analysis and visualization
+• File ready for immediate business use
 
-def create_worksheet_styles():
-    # Define professional styles
+=== MANDATORY EXECUTION SEQUENCE ===
 
-def create_executive_summary(wb, data):
-    # Create executive summary with key metrics
+**STEP 1: Environment Setup (30 seconds)**
+```python
+# RestrictedPythonTools automatically installs packages when needed
+# Just use run_python_code() - packages will be auto-installed
+import pandas as pd
+import openpyxl
+print("Packages will be auto-installed by RestrictedPythonTools")
+```
 
-def create_financial_statements(wb, data):
-    # Create income statement, balance sheet, cash flow
+**STEP 2: Revenue Data Loading (30 seconds)**
+- read_file('arranged_financial_data.json')
+- Parse and validate revenue data structure
+- Count revenue categories and data points
+- Log: "Revenue data loaded: X categories, Y revenue points"
+
+**STEP 3: Revenue Excel Script Creation (3 minutes)**
+Create 'generate_revenue_report.py' with this EXACT structure:
+
+```python
+#!/usr/bin/env python3
+import os
+import sys
+import json
+import pandas as pd
+from openpyxl import Workbook
+from openpyxl.styles import Font, PatternFill, Border, Side, Alignment
+from datetime import datetime
+import logging
 
-def add_charts_and_visualizations(wb, data):
-    # Add appropriate charts to worksheets
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
 
-def generate_financial_report():
+def main():
     try:
-        data = load_financial_data()
+        # Load revenue data
+        logger.info('Loading revenue data from arranged_financial_data.json')
+        with open('arranged_financial_data.json', 'r', encoding='utf-8') as f:
+            revenue_data = json.load(f)
+        
+        # Create professional workbook
+        logger.info('Creating revenue analysis workbook')
         wb = Workbook()
-        create_worksheet_styles()
-        create_executive_summary(wb, data)
-        create_financial_statements(wb, data)
-        add_charts_and_visualizations(wb, data)
+        wb.remove(wb.active)  # Remove default sheet
+        
+        # Define professional styling
+        header_font = Font(bold=True, color='FFFFFF', size=12)
+        header_fill = PatternFill(start_color='1F4E79', end_color='1F4E79', fill_type='solid')
+        data_font = Font(size=11)
+        
+        # Process each revenue category
+        revenue_categories = ['Company_Overview', 'Total_Revenue', 'Segment_Revenue', 'Regional_Revenue', 'Data_Quality']
         
-        # Save with timestamp
-        timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
-        filename = f'Financial_Report_{timestamp}.xlsx'
+        for category_name in revenue_categories:
+            if category_name in revenue_data:
+                logger.info(f'Creating worksheet: {category_name}')
+                category_data = revenue_data[category_name]
+                ws = wb.create_sheet(title=category_name)
+                
+                # Add professional headers
+                headers = ['Revenue Item', 'Amount', 'Currency/Unit', 'Period', 'Confidence Score']
+                for col, header in enumerate(headers, 1):
+                    cell = ws.cell(row=1, column=col, value=header)
+                    cell.font = header_font
+                    cell.fill = header_fill
+                    cell.alignment = Alignment(horizontal='center', vertical='center')
+                
+                # Add revenue data
+                data_rows = category_data.get('data', [])
+                for row_idx, data_row in enumerate(data_rows, 2):
+                    ws.cell(row=row_idx, column=1, value=data_row.get('item', '')).font = data_font
+                    ws.cell(row=row_idx, column=2, value=data_row.get('value', '')).font = data_font
+                    ws.cell(row=row_idx, column=3, value=data_row.get('unit', '')).font = data_font
+                    ws.cell(row=row_idx, column=4, value=data_row.get('period', '')).font = data_font
+                    ws.cell(row=row_idx, column=5, value=data_row.get('confidence', '')).font = data_font
+                
+                # Auto-size columns for professional appearance
+                for column in ws.columns:
+                    max_length = 0
+                    column_letter = column[0].column_letter
+                    for cell in column:
+                        try:
+                            if len(str(cell.value or '')) > max_length:
+                                max_length = len(str(cell.value or ''))
+                        except:
+                            pass
+                    adjusted_width = min(max(max_length + 2, 15), 50)
+                    ws.column_dimensions[column_letter].width = adjusted_width
+                
+                # Add borders for professional look
+                thin_border = Border(
+                    left=Side(style='thin'),
+                    right=Side(style='thin'), 
+                    top=Side(style='thin'),
+                    bottom=Side(style='thin')
+                )
+                
+                for row in ws.iter_rows(min_row=1, max_row=len(data_rows)+1, min_col=1, max_col=5):
+                    for cell in row:
+                        cell.border = thin_border
+        
+        # Save with professional filename
+        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+        filename = f'Revenue_Analysis_Report_{timestamp}.xlsx'
         wb.save(filename)
-        logging.info(f'Report saved as {filename}')
-        return filename
+        logger.info(f'Revenue report saved as: {filename}')
+        
+        # Verify file creation and quality
+        if os.path.exists(filename):
+            file_size = os.path.getsize(filename)
+            if file_size > 5000:  # Minimum 5KB
+                logger.info(f'SUCCESS: Revenue report created successfully')
+                logger.info(f'File: {filename} ({file_size:,} bytes)')
+                logger.info(f'Worksheets: {len(wb.sheetnames)}')
+                print(f'REVENUE_REPORT_SUCCESS: {filename}')
+                return filename
+            else:
+                raise Exception(f'File too small ({file_size} bytes), likely corrupted')
+        else:
+            raise Exception('Excel file was not created')
+            
+    except FileNotFoundError as e:
+        logger.error(f'Revenue data file not found: {str(e)}')
+        sys.exit(1)
+    except json.JSONDecodeError as e:
+        logger.error(f'Invalid JSON in revenue data: {str(e)}')
+        sys.exit(1)
     except Exception as e:
-        logging.error(f'Error generating report: {e}')
-        raise
+        logger.error(f'Error creating revenue report: {str(e)}')
+        import traceback
+        logger.error(f'Traceback: {traceback.format_exc()}')
+        sys.exit(1)
 
 if __name__ == '__main__':
-    generate_financial_report()
+    result = main()
+    print(f'COMPLETED: {result}')
 ```
 
-=== EXECUTION STEPS ===
-1. Read and analyze 'arranged_financial_data.json' structure
-2. Install required packages: pip_install_package('openpyxl')
-3. Create comprehensive Python script with error handling
-4. Save script using save_file tool
-5. Execute using run_shell_command('python generate_excel_report.py 2>&1')
-6. Verify file creation with list_files
-7. Validate file size and integrity
-8. Report execution results and any issues
+**STEP 4: Script Execution with RestrictedPythonTools (2 minutes)**
+- Use run_python_code([complete_script]) for direct execution with auto-healing
+- OR save_python_file('generate_revenue_report.py', [complete_script]) + run_shell_command('python generate_revenue_report.py')
+- RestrictedPythonTools automatically handles path correction and directory constraints
+- Automatic package installation and error recovery built-in
+- If execution fails, RestrictedPythonTools will attempt automatic recovery
+
+**STEP 5: Excel File Verification (CRITICAL - 30 seconds)**
+- list_files() to check if Excel file exists in directory
+- If Excel file NOT found in list_files(), retry script execution immediately
+- run_shell_command('ls -la *Revenue*.xlsx') for detailed file info
+- run_shell_command('du -h *Revenue*.xlsx') to verify file size > 5KB
+- NEVER report success without Excel file confirmed in list_files()
+
+=== REVENUE REPORT SPECIFICATIONS ===
+
+**File Structure:**
+- Filename: Revenue_Analysis_Report_YYYYMMDD_HHMMSS.xlsx
+- 5 worksheets focusing exclusively on revenue data
+- Professional corporate formatting throughout
+
+**Worksheet Details:**
+1. **Company_Overview** - Company info, document metadata
+2. **Total_Revenue** - Consolidated revenue figures and totals
+3. **Segment_Revenue** - Revenue by business segment/division
+4. **Regional_Revenue** - Revenue by geographic region
+5. **Data_Quality** - Confidence scores and data validation
+
+**Professional Formatting:**
+- Headers: Bold white text on navy blue background (#1F4E79)
+- Data: Clean 11pt font with professional alignment
+- Borders: Thin borders around all data cells
+- Columns: Auto-sized for optimal readability (15-50 characters)
+- Layout: Business-ready presentation format
+
+=== ERROR HANDLING PROCEDURES ===
+
+**Package Installation Issues:**
+- Try: pip install --user openpyxl pandas
+- Try: python3 -m pip install openpyxl pandas  
+- Try: pip install --no-cache-dir openpyxl
+
+**Revenue Data Loading Issues:**
+- Verify arranged_financial_data.json exists
+- Check JSON syntax and structure
+- Ensure revenue categories are present
+
+**Excel Generation Issues:**
+- Log exact openpyxl error messages
+- Try simplified formatting if complex formatting fails
+- Check file write permissions in directory
+- Verify Python version compatibility
+
+**File Verification Issues:**
+- Check file exists and has reasonable size (>5KB)
+- Verify Excel file can be opened without corruption
+- Confirm all expected worksheets are present
 
 === SUCCESS CRITERIA ===
-✅ Excel file created with timestamp filename
-✅ File size >10KB (indicates substantial content)
-✅ All worksheets present and formatted professionally
-✅ Charts and visualizations included
-✅ No execution errors in logs
-✅ Data accurately transferred from JSON to Excel
-
-=== ERROR HANDLING ===
-If issues occur:
-• Log detailed error information
-• Identify root cause (data, formatting, or execution)
-• Implement fixes and retry
-• Provide clear status updates
-
-Generate the comprehensive Excel report now.
+Revenue Excel generation is successful ONLY if:
+✓ openpyxl package installed without errors
+✓ Revenue data loaded and parsed successfully
+✓ Python script executed without errors  
+✓ Excel file created with proper filename format
+✓ File size > 5KB indicating data was written
+✓ All 5 revenue worksheets present and populated
+✓ Professional formatting applied consistently
+✓ File opens without corruption in Excel
+
+=== PROFESSIONAL FEATURES ===
+Your Excel report MUST include:
+- **Corporate Design**: Professional navy blue headers with white text
+- **Business Layout**: Clean, executive-ready formatting
+- **Data Integrity**: All original revenue values preserved exactly
+- **User Experience**: Auto-sized columns, proper alignment, clear borders
+- **File Management**: Timestamped filename for version control
+- **Quality Assurance**: Comprehensive error handling and validation
+
+=== FINAL VALIDATION CHECKLIST ===
+Before reporting success, verify:
+□ All required packages installed successfully
+□ Revenue data JSON loaded and parsed correctly
+□ Python script saved and executed without errors
+□ Excel file created with timestamped filename
+□ File size indicates successful data population (>5KB)
+□ All 5 revenue worksheets present and properly named
+□ Revenue data populated correctly in each worksheet
+□ Professional formatting applied consistently
+□ No execution errors or warnings in output
+□ File can be opened by Excel applications
+
+Execute now. Focus EXCLUSIVELY on revenue data visualization. Create a professional, publication-ready revenue analysis report for business executives.
+

prompts/workflow/data_arrangement.txt

@@ -1,34 +1,35 @@
-You are given raw, extracted financial data. Your task is to reorganize it and prepare it for Excel-based reporting.
+ORGANIZE REVENUE DATA FOR EXCEL
 
-========== WHAT TO DELIVER ==========
-• A single JSON object saved as arranged_financial_data.json
-• Fields required: categories, headers, metadata
+Your task: Organize the provided revenue data into a JSON file with 5 categories.
 
-========== HOW TO ORGANIZE ==========
-Create distinct, Excel-ready categories (one worksheet each) for logical grouping of financial data. Examples include:
-1. Income Statement Data
-2. Balance Sheet Data
-3. Cash Flow Data
-4. Company Information / General Data
+CATEGORIES TO CREATE:
+1. Company_Overview - Company details and metadata
+2. Total_Revenue - All total/consolidated revenue figures  
+3. Segment_Revenue - Revenue by business segment/division
+4. Regional_Revenue - Revenue by geographic region
+5. Data_Quality - Confidence scores and extraction notes
 
-========== STEP-BY-STEP ==========
-1. Map every data point into the most appropriate category above.
-2. For each category, identify and include all necessary headers for an Excel template, such as years, company names, financial line item names, and units of measurement (e.g., "in millions").
-3. Ensure data integrity by not modifying, calculating, or analyzing the original data values.
-4. Preserve original data formats and units.
-5. Organize data in a tabular format suitable for direct Excel import.
-6. Include metadata about data sources and reporting periods where available.
-7. Assemble everything into the JSON schema described under "WHAT TO DELIVER."
-8. Save the JSON as arranged_financial_data.json via save_file.
-9. Use list_files to confirm the file exists, then read_file to validate its content.
-10. If the file is missing or malformed, fix the issue and repeat steps 8 – 9.
-11. Only report success after the file passes both existence and content checks.
+JSON STRUCTURE:
+{
+  "Category_Name": {
+    "headers": {"Item": "...", "Value": "...", "Unit": "...", "Period": "...", "Confidence": "..."},
+    "data": [{"item": "...", "value": "...", "unit": "...", "period": "...", "confidence": "..."}],
+    "metadata": {"description": "...", "data_count": "...", "quality_score": "..."}
+  }
+}
 
-========== IMPORTANT RESTRICTIONS ==========
-- Never perform any analysis on the data.
-- Do not calculate ratios, growth rates, or trends.
-- Do not provide insights or interpretations.
-- Do not modify the actual data values.
-- Focus solely on organization and proper formatting.
+STEPS:
+1. Parse the revenue data below
+2. Map each data point to the correct category
+3. Create the JSON structure
+4. save_file('arranged_financial_data.json', json_content)
+5. list_files() to verify the file exists
+6. If file not found, retry save_file() once
 
-Extracted Data: {extracted_data}
+RULES:
+- Use EXACT values from the input data (no modifications)
+- Use EXACT company name, currency, and period from input
+- Focus ONLY on revenue data (ignore expenses, assets, etc.)
+
+The revenue data to organize:
+{extracted_data}

prompts/workflow/data_arrangement_1.txt

@@ -0,0 +1,160 @@
+REVENUE DATA ORGANIZATION TASK
+
+=== YOUR MISSION ===
+Organize ONLY the extracted revenue data and prepare it for Excel-based reporting.
+Focus exclusively on revenue-related data - ignore all other financial information.
+You are using gemini-2.5-pro with thinking budget optimization.
+
+🚨 CRITICAL: You MUST use the ACTUAL extracted data provided below. 
+NEVER create fake/sample data. Use EXACT company names, values, periods, and currencies from the provided extraction.
+
+=== EXTRACTED REVENUE DATA ===
+(Data will be provided below)
+
+=== WHAT TO DELIVER ===
+• A single JSON file saved as arranged_financial_data.json
+• ONLY 5 revenue-focused categories for Excel worksheets
+• Preserve all original revenue data exactly
+
+=== REVENUE-FOCUSED ORGANIZATION ===
+Create EXACTLY these 5 Excel-ready categories (revenue only):
+
+**1. Company_Overview**
+- Company name, document type, reporting period
+- Currency, extraction date, data quality summary
+
+**2. Total_Revenue** 
+- Consolidated revenue figures
+- Total revenue metrics from income statement
+- Year-over-year revenue if available
+
+**3. Segment_Revenue**
+- Revenue by business segment/division
+- Product vs service revenue breakdowns  
+- Business unit revenue performance
+
+**4. Regional_Revenue**
+- Revenue by geographic region
+- Country-specific revenue data
+- International vs domestic revenue splits
+
+**5. Data_Quality**
+- Confidence scores for each revenue data point
+- Source information and validation notes
+- Revenue extraction metadata
+
+=== STEP-BY-STEP PROCESS ===
+
+**Step 1: Revenue Data Analysis (1 minute)**
+- Parse the extracted revenue data completely - USE THE ACTUAL PROVIDED DATA
+- Count total revenue data points from the PROVIDED extraction
+- Identify revenue categories (total, segment, regional) from ACTUAL data
+- Filter out all non-revenue data points from PROVIDED data
+- NEVER create sample/fake data - ONLY use the provided extracted data
+- Log: "Revenue analysis complete: X revenue points identified from PROVIDED data"
+
+**Step 2: Revenue Data Mapping (2 minutes)**  
+- Map ONLY the PROVIDED revenue data points to appropriate categories:
+  - Total Revenue from PROVIDED data → "Total_Revenue" 
+  - Segment/Division Revenue from PROVIDED data → "Segment_Revenue"
+  - Geographic/Regional Revenue from PROVIDED data → "Regional_Revenue"
+  - Company metadata from PROVIDED data → "Company_Overview"
+  - Confidence/source data from PROVIDED data → "Data_Quality"
+- IGNORE all non-revenue data (expenses, assets, liabilities, etc.)
+- CRITICAL: Use EXACT values, company names, periods, and currencies from the PROVIDED extracted data
+
+**Step 3: JSON Structure Creation (2 minutes)**
+Create this EXACT structure using ONLY the PROVIDED extracted data:
+
+```json
+{
+  "[Category_Name]": {
+    "headers": {
+      "Item": "Revenue Item",
+      "Value": "Amount", 
+      "Unit": "Currency/Scale",
+      "Period": "Reporting Period",
+      "Confidence": "Accuracy Score"
+    },
+    "data": [
+      {
+        "item": "[EXACT field name from PROVIDED data]",
+        "value": "[EXACT value from PROVIDED data - no modifications]",
+        "unit": "[EXACT unit from PROVIDED data]", 
+        "period": "[EXACT period from PROVIDED data]",
+        "confidence": "[EXACT confidence from PROVIDED data]"
+      }
+    ],
+    "metadata": {
+      "description": "[Category description]",
+      "data_count": "[Number of items from PROVIDED data]",
+      "quality_score": "[Average confidence from PROVIDED data]"
+    }
+  }
+}
+```
+
+**CRITICAL RULES FOR DATA USAGE:**
+- Use EXACT company name from provided metadata (e.g., "Deutsche Telekom AG")
+- Use EXACT currency from provided data (e.g., "EUR")
+- Use EXACT reporting period from provided data (e.g., "FY 2023")
+- Use EXACT revenue values from provided data (e.g., "111,985 million")
+- NEVER create fake/sample data like "Global Corp", "Q2 2025", or made-up numbers
+
+**Step 4: File Operations with Verification (1 minute)**
+- save_file('arranged_financial_data.json', complete_json_structure)
+- list_files() to verify file exists - CRITICAL VERIFICATION STEP
+- If file NOT found in list_files(), retry save_file() operation immediately
+- read_file('arranged_financial_data.json') to validate JSON syntax
+- If any step fails, retry up to 3 times total
+- Log: "Revenue data organization complete: file saved and validated"
+
+=== DATA PRESERVATION RULES ===
+CRITICAL - You MUST follow these rules exactly:
+- **NEVER** modify original revenue values
+- **NEVER** perform calculations or analysis
+- **NEVER** interpret or add insights  
+- **NEVER** change currency units or scales
+- **ONLY** organize revenue data for Excel import
+- **IGNORE** all non-revenue financial data completely
+
+=== REVENUE DATA VALIDATION ===
+Before saving, verify:
+- Company name preserved exactly from extraction
+- Total revenue data properly categorized
+- Segment revenue breakdowns organized logically
+- Regional revenue data grouped appropriately
+- All confidence scores preserved
+- All original values unchanged
+
+=== FILE OPERATIONS SEQUENCE ===
+Execute in EXACT order with verification:
+1. **save_file('arranged_financial_data.json', json_content)**
+2. **list_files() - CRITICAL VERIFICATION STEP**
+   - Check if 'arranged_financial_data.json' appears in the file list
+   - If file NOT found, retry save_file() operation immediately
+3. **read_file('arranged_financial_data.json')** - validate JSON syntax
+4. **Retry up to 3 times total if any step fails**
+   - NEVER proceed without file confirmation in list_files()
+
+=== SUCCESS CRITERIA ===
+Organization is successful ONLY if:
+✓ arranged_financial_data.json saved and confirmed in list_files()
+✓ File exists and is readable via read_file()
+✓ JSON syntax is valid and well-formed
+✓ All 5 revenue categories are present (even if empty)
+✓ Every revenue data point mapped to exactly one category
+✓ No original revenue values modified
+✓ All non-revenue data filtered out
+✓ File validation passes completely
+
+=== ERROR HANDLING ===
+If you encounter issues:
+- **Empty revenue data**: Create categories with "No revenue data available" 
+- **Invalid JSON**: Fix syntax errors and retry save
+- **File save fails**: Try different filename and retry
+- **Missing categories**: Create empty category with metadata
+- **Non-revenue data**: Filter out completely, focus only on revenue
+
+Execute now. Focus EXCLUSIVELY on revenue data organization. Preserve all revenue values exactly as extracted.
+

prompts/workflow/data_extraction.txt

@@ -1,58 +1,135 @@
-You are a financial data extraction specialist analyzing the document at: {file_path}
-
-=== EXTRACTION APPROACH ===
-Use a systematic 5-phase approach: Document Analysis → Critical Data → Standard Financials → Advanced Metrics → Quality Assurance
-
-=== PHASE 1: DOCUMENT ANALYSIS ===
-First, quickly identify:
-• Document type (Annual Report, 10-K, 10-Q, Quarterly Report, etc.)
-• Company name and ticker symbol
-• Reporting period and fiscal year
-• Currency and unit scales (millions/thousands)
-• Location of key financial statements
-
-=== PHASE 2: CRITICAL DATA (Must Extract) ===
-🔴 Company Essentials:
-• Official company name and ticker
-• Reporting period and currency
-• Document type and audit status
-
-🔴 Core Performance:
-• Total Revenue/Net Sales
-• Net Income/Profit
-• Total Assets
-• Total Shareholders' Equity
-• Basic Earnings Per Share (EPS)
-
-=== PHASE 3: STANDARD FINANCIALS (High Priority) ===
-📊 Income Statement: Revenue breakdown, COGS, gross profit, operating expenses, operating income, interest, taxes, diluted EPS
-💰 Balance Sheet: Current/non-current assets, current/non-current liabilities, equity components
-💸 Cash Flow: Operating, investing, financing cash flows, capex, free cash flow
-
-=== PHASE 4: ADVANCED METRICS (If Available) ===
-📈 Financial Ratios: Margins, returns (ROE/ROA), liquidity ratios, leverage ratios
-👥 Operational Data: Employee count, locations, customer metrics, production volumes
-📋 Supplementary: Dividends, buybacks, guidance, one-time items
-
-=== PHASE 5: QUALITY ASSURANCE ===
-• Validate Balance Sheet equation (Assets = Liabilities + Equity)
-• Assign confidence scores: 1.0 (clearly stated) to 0.4 (unclear)
-• Flag missing critical data with explanations
-• Note any unusual values or inconsistencies
+REVENUE-FOCUSED FINANCIAL DATA EXTRACTION
+
+=== DOCUMENT TO ANALYZE ===
+File: {file_path}
+(Document will be provided directly to you for analysis)
+
+=== YOUR MISSION ===
+Extract ONLY revenue-related financial data from the provided document with 100% accuracy.
+Focus exclusively on company name and revenue data - ignore all other financial metrics.
+You are using gemini-2.5-pro with thinking budget optimization.
+
+=== WHAT TO EXTRACT (REVENUE ONLY) ===
+
+**MANDATORY (Must Extract):**
+1. **Company Name** - Official legal company name
+2. **Total Revenue** - Consolidated revenue/net sales for most recent period
+3. **Document Type** - 10-K, 10-Q, Annual Report, Quarterly Report, etc.
+4. **Reporting Period** - FY 2023, Q1 2024, etc.
+5. **Currency** - USD, EUR, etc.
+
+**HIGH VALUE (Extract if clearly present):**
+6. **Segment Revenue** - Revenue by business segment/division/product line
+7. **Regional Revenue** - Revenue by geographic region/country
+
+**IGNORE COMPLETELY:**
+- Net income, profit, losses
+- Assets, liabilities, equity
+- Cash flow data
+- Expenses, costs, operating income
+- Balance sheet items
+- Ratios, per-share metrics
+- Non-financial data
+
+=== SYSTEMATIC EXTRACTION PROCESS ===
+
+**Step 1: Document Structure Analysis**
+- Scan document to understand structure and layout
+- Identify document type and reporting period
+- Locate revenue-related sections (Income Statement, Segment Reporting, Geographic Data)
+
+**Step 2: Company Identification** 
+- Extract official company name from document header/title
+- Verify name consistency throughout document
+- Use parent company name if multiple entities present
+
+**Step 3: Total Revenue Extraction (CRITICAL)**
+- Find consolidated revenue figure for most recent period
+- Look in: Consolidated Statements of Operations, Income Statement
+- Search terms: "Revenue", "Net Sales", "Total Revenue", "Net Revenue"
+- Record exact value with currency and time period
+
+**Step 4: Segment Revenue Analysis**
+- Locate segment reporting section (usually separate section after financial statements)  
+- Extract revenue by business segment, division, or product line
+- Common segments: Products, Services, Geographic regions, Business units
+- Ensure segment revenues sum to total revenue for validation
+
+**Step 5: Regional Revenue Analysis** 
+- Find geographic revenue breakdown section
+- Look for revenue by: Americas, EMEA, APAC, US vs International, specific countries
+- Extract revenue figures for major geographic regions
+- Validate regional totals match consolidated revenue
+
+**Step 6: Data Validation**
+- Verify company name is not empty
+- Confirm total revenue has high confidence score (>0.7)
+- Check that segment/regional breakdowns sum to total
+- Ensure all mandatory fields are extracted
+
+=== CONFIDENCE SCORING (REVENUE DATA ONLY) ===
+- **1.0**: Revenue clearly stated in financial table with proper labels
+- **0.8**: Revenue stated in structured text with clear context
+- **0.6**: Revenue derived from segment/regional totals
+- **0.4**: Revenue estimated or context somewhat unclear
+- **0.2**: Revenue barely visible or questionable source
+- **0.0**: Revenue not found or completely unclear
 
 === OUTPUT REQUIREMENTS ===
-Return structured data using ExtractedFinancialData model:
-• company_name: Official company name
-• document_type: Type of document analyzed
-• reporting_period: Fiscal period (e.g., 'FY 2023')
-• data_points: Array with field_name, value, category, period, unit, confidence
-• summary: 2-3 sentence summary of key findings
-
-=== EXTRACTION TIPS ===
-• Look in financial tables first, then notes, then text
-• Watch for footnotes and accounting changes
-• Note restatements or discontinued operations
-• Pay attention to scale indicators (millions/thousands)
-• Extract multiple periods when available
-
-Document to analyze: {file_path}
+Return ExtractedFinancialData with ONLY revenue-related data:
+
+```json
+{
+  "company_name": "[Official Company Name]",
+  "document_type": "[10-K|10-Q|Annual Report|etc.]",
+  "reporting_period": "[FY 2023|Q1 2024|etc.]", 
+  "currency": "[USD|EUR|etc.]",
+  "data_points": [
+    {
+      "field_name": "Total Revenue",
+      "value": "$50.3 billion", 
+      "category": "Revenue",
+      "period": "FY 2023",
+      "unit": "USD billions",
+      "confidence": 1.0
+    },
+    {
+      "field_name": "Product Revenue", 
+      "value": "$30.2 billion",
+      "category": "Segment Revenue", 
+      "period": "FY 2023",
+      "unit": "USD billions",
+      "confidence": 0.9
+    },
+    {
+      "field_name": "Americas Revenue",
+      "value": "$25.1 billion",
+      "category": "Regional Revenue",
+      "period": "FY 2023", 
+      "unit": "USD billions",
+      "confidence": 0.8
+    }
+  ],
+  "summary": "[2-3 sentences describing key revenue findings and trends]"
+}
+```
+
+=== SUCCESS CRITERIA ===
+Extraction is successful ONLY if:
+✓ Company name extracted (never empty)
+✓ Total revenue extracted with confidence > 0.5
+✓ Document type and period identified
+✓ Currency specified
+✓ All data points are revenue-related only
+✓ Summary focuses on revenue insights (2-3 sentences)
+✓ Segment/regional data sums to total (if present)
+
+=== REVENUE EXTRACTION STRATEGY ===
+1. **Income Statement First** - Look for consolidated revenue in primary financial statements
+2. **Segment Section Second** - Find detailed segment revenue breakdowns  
+3. **Geographic Section Third** - Locate regional revenue data
+4. **Management Discussion** - Check for revenue highlights and explanations
+5. **Tables Over Text** - Prioritize tabular data over narrative mentions
+
+**Remember**: Focus EXCLUSIVELY on revenue data. Ignore all other financial metrics. Your goal is 100% accuracy on revenue extraction with proper segment and regional breakdowns.
+

utils/restricted_python_tools.py

@@ -0,0 +1,470 @@
+"""
+RestrictedPythonTools - Self-Healing Python Execution with Shell Backend
+
+This toolkit provides Python code execution with built-in directory constraints,
+path auto-correction, and self-healing capabilities. Uses RestrictedShellTools
+as the backend execution engine, mirroring Claude Code's architecture.
+"""
+
+import os
+import re
+import ast
+import sys
+import json
+import time
+import uuid
+import tempfile
+from pathlib import Path
+from typing import Optional, Dict, Any, List
+from agno.tools import Toolkit
+from agno.utils.log import logger
+
+from .shell_toolkit import RestrictedShellTools
+
+
+class RestrictedPythonTools(Toolkit):
+    """
+    Self-healing Python execution toolkit with directory constraints.
+    
+    Uses RestrictedShellTools as backend for secure, constrained Python execution.
+    Includes automatic path correction, package installation, and error recovery.
+    """
+    
+    def __init__(self, base_dir: Optional[Path] = None, **kwargs):
+        """
+        Initialize the restricted Python toolkit.
+        
+        Args:
+            base_dir: Base directory to constrain all Python operations to
+            **kwargs: Additional arguments passed to parent Toolkit
+        """
+        self.base_dir = Path(base_dir) if base_dir else Path.cwd()
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Initialize backend tools
+        self.shell_tools = RestrictedShellTools(base_dir=self.base_dir)
+        
+        # Track installed packages to avoid redundant installations
+        self.installed_packages = set()
+        
+        # Initialize toolkit with Python execution functions
+        super().__init__(
+            name="restricted_python_tools",
+            tools=[
+                self.run_python_code,
+                self.install_package,
+                self.save_python_file,
+                self.list_python_files,
+                self.validate_python_syntax
+            ],
+            **kwargs
+        )
+        
+        logger.info(f"RestrictedPythonTools initialized with base_dir: {self.base_dir}")
+    
+    def run_python_code(self, code: str, timeout: int = 120) -> str:
+        """
+        Execute Python code with self-healing and directory constraints.
+        
+        Args:
+            code (str): Python code to execute
+            timeout (int): Maximum execution time in seconds
+            
+        Returns:
+            str: Output from code execution or error message
+        """
+        try:
+            # Step 1: Auto-correct and heal the code
+            healed_code = self._heal_python_code(code)
+            
+            # Step 2: Validate syntax before execution
+            syntax_result = self.validate_python_syntax(healed_code)
+            if "Error" in syntax_result:
+                return f"Syntax Error: {syntax_result}"
+            
+            # Step 3: Extract and auto-install required packages
+            self._auto_install_packages(healed_code)
+            
+            # Step 4: Create temporary Python file
+            temp_filename = f"temp_script_{uuid.uuid4().hex[:8]}.py"
+            temp_filepath = self.base_dir / temp_filename
+            
+            try:
+                # Save healed code to temporary file
+                with open(temp_filepath, 'w', encoding='utf-8') as f:
+                    f.write(healed_code)
+                
+                logger.info(f"Executing Python code via shell backend: {temp_filename}")
+                
+                # Step 5: Execute via RestrictedShellTools backend
+                execution_command = f"python3 {temp_filename}"
+                result = self.shell_tools.run_shell_command(execution_command, timeout=timeout)
+                
+                # Step 6: Check for common errors and attempt recovery
+                if self._has_execution_errors(result):
+                    recovery_result = self._attempt_error_recovery(healed_code, result, temp_filename, timeout)
+                    if recovery_result:
+                        result = recovery_result
+                
+                return result
+                
+            finally:
+                # Cleanup temporary file
+                if temp_filepath.exists():
+                    temp_filepath.unlink()
+                    
+        except Exception as e:
+            error_msg = f"Error executing Python code: {str(e)}"
+            logger.error(error_msg)
+            return error_msg
+    
+    def _heal_python_code(self, code: str) -> str:
+        """
+        Auto-correct common path and directory issues in Python code.
+        
+        Args:
+            code (str): Original Python code
+            
+        Returns:
+            str: Healed Python code with corrected paths
+        """
+        healed_code = code
+        
+        # Path correction patterns
+        path_corrections = [
+            # Fix relative paths that go outside base directory
+            (r'\.\./', ''),
+            (r'\.\./\.\./', ''),
+            (r'\.\.\\', ''),
+            
+            # Convert absolute paths to relative paths within base directory
+            (r'["\']\/[^"\']*\/([^"\'\/]+\.(xlsx?|csv|json|txt|py))["\']', r'"\1"'),
+            
+            # Fix common pandas path issues
+            (r'pd\.to_excel\(["\'][^"\']*\/([^"\'\/]+\.xlsx?)["\']', r'pd.to_excel("\1"'),
+            (r'pd\.read_excel\(["\'][^"\']*\/([^"\'\/]+\.xlsx?)["\']', r'pd.read_excel("\1"'),
+            (r'pd\.to_csv\(["\'][^"\']*\/([^"\'\/]+\.csv)["\']', r'pd.to_csv("\1"'),
+            
+            # Fix file operations
+            (r'open\(["\'][^"\']*\/([^"\'\/]+)["\']', r'open("\1"'),
+            (r'with open\(["\'][^"\']*\/([^"\'\/]+)["\']', r'with open("\1"'),
+        ]
+        
+        for pattern, replacement in path_corrections:
+            healed_code = re.sub(pattern, replacement, healed_code)
+        
+        # Add working directory insurance at the beginning
+        directory_insurance = f"""
+import os
+import sys
+
+# Ensure we're in the correct working directory
+base_dir = r'{self.base_dir}'
+if os.getcwd() != base_dir:
+    os.chdir(base_dir)
+    print(f"Working directory corrected to: {{os.getcwd()}}")
+
+"""
+        
+        # Add directory insurance to the beginning of the code
+        healed_code = directory_insurance + healed_code
+        
+        logger.debug(f"Code healing applied - original length: {len(code)}, healed length: {len(healed_code)}")
+        return healed_code
+    
+    def _extract_required_packages(self, code: str) -> List[str]:
+        """
+        Extract package names from import statements in Python code.
+        
+        Args:
+            code (str): Python code to analyze
+            
+        Returns:
+            List[str]: List of package names that need to be installed
+        """
+        packages = set()
+        
+        # Built-in modules that don't need installation
+        builtin_modules = {
+            'os', 'sys', 'json', 'time', 'datetime', 'uuid', 'tempfile', 
+            're', 'ast', 'pathlib', 'math', 'random', 'subprocess', 
+            'collections', 'itertools', 'functools', 'logging', 'io',
+            'csv', 'xml', 'urllib', 'http', 'email', 'sqlite3'
+        }
+        
+        # Common package mappings (import name -> pip package name)
+        package_mappings = {
+            'pandas': 'pandas',
+            'numpy': 'numpy', 
+            'openpyxl': 'openpyxl',
+            'xlsxwriter': 'xlsxwriter',
+            'matplotlib': 'matplotlib',
+            'seaborn': 'seaborn',
+            'plotly': 'plotly',
+            'requests': 'requests',
+            'beautifulsoup4': 'beautifulsoup4',
+            'bs4': 'beautifulsoup4',
+            'sklearn': 'scikit-learn',
+            'cv2': 'opencv-python',
+            'PIL': 'Pillow',
+            'yaml': 'PyYAML',
+        }
+        
+        # Extract import statements using regex
+        import_patterns = [
+            r'^import\s+([a-zA-Z_][a-zA-Z0-9_]*)',
+            r'^from\s+([a-zA-Z_][a-zA-Z0-9_]*)\s+import',
+        ]
+        
+        for line in code.split('\n'):
+            line = line.strip()
+            for pattern in import_patterns:
+                match = re.match(pattern, line)
+                if match:
+                    package_name = match.group(1)
+                    
+                    # Skip built-in modules
+                    if package_name in builtin_modules:
+                        continue
+                    
+                    # Map to pip package name if known
+                    pip_package = package_mappings.get(package_name, package_name)
+                    packages.add(pip_package)
+        
+        return list(packages)
+    
+    def _auto_install_packages(self, code: str) -> None:
+        """
+        Automatically install required packages for the Python code.
+        
+        Args:
+            code (str): Python code to analyze for package requirements
+        """
+        required_packages = self._extract_required_packages(code)
+        
+        for package in required_packages:
+            if package not in self.installed_packages:
+                logger.info(f"Auto-installing package: {package}")
+                install_result = self.install_package(package)
+                if "successfully" in install_result.lower():
+                    self.installed_packages.add(package)
+                else:
+                    logger.warning(f"Failed to install package {package}: {install_result}")
+    
+    def _has_execution_errors(self, result: str) -> bool:
+        """
+        Check if execution result contains errors that might be recoverable.
+        
+        Args:
+            result (str): Execution result to check
+            
+        Returns:
+            bool: True if recoverable errors are detected
+        """
+        error_indicators = [
+            "ModuleNotFoundError",
+            "ImportError",
+            "FileNotFoundError",
+            "PermissionError",
+            "No such file or directory",
+        ]
+        
+        return any(error in result for error in error_indicators)
+    
+    def _attempt_error_recovery(self, code: str, error_result: str, temp_filename: str, timeout: int) -> Optional[str]:
+        """
+        Attempt to recover from execution errors.
+        
+        Args:
+            code (str): Original code that failed
+            error_result (str): Error message from failed execution
+            temp_filename (str): Temporary file name used
+            timeout (int): Execution timeout
+            
+        Returns:
+            Optional[str]: Recovery result if successful, None if recovery failed
+        """
+        try:
+            # Recovery attempt 1: Install missing packages
+            if "ModuleNotFoundError" in error_result or "ImportError" in error_result:
+                logger.info("Attempting recovery: Installing missing packages")
+                
+                # Extract package name from error message
+                missing_package_match = re.search(r"No module named '([^']+)'", error_result)
+                if missing_package_match:
+                    missing_package = missing_package_match.group(1)
+                    install_result = self.install_package(missing_package)
+                    
+                    if "successfully" in install_result.lower():
+                        logger.info(f"Recovery successful: Installed {missing_package}")
+                        # Retry execution
+                        retry_result = self.shell_tools.run_shell_command(f"python3 {temp_filename}", timeout=timeout)
+                        return retry_result
+            
+            # Recovery attempt 2: Fix file path issues
+            if "FileNotFoundError" in error_result or "No such file or directory" in error_result:
+                logger.info("Attempting recovery: Fixing file path issues")
+                
+                # Create any missing directories that might be referenced
+                self.shell_tools.run_shell_command("mkdir -p data reports output")
+                
+                # Retry execution
+                retry_result = self.shell_tools.run_shell_command(f"python3 {temp_filename}", timeout=timeout)
+                return retry_result
+            
+        except Exception as e:
+            logger.error(f"Error recovery failed: {str(e)}")
+        
+        return None
+    
+    def install_package(self, package_name: str) -> str:
+        """
+        Install a Python package using pip via shell backend.
+        
+        Args:
+            package_name (str): Name of the package to install
+            
+        Returns:
+            str: Installation result message
+        """
+        try:
+            logger.info(f"Installing Python package: {package_name}")
+            
+            # Try multiple installation methods
+            install_commands = [
+                f"pip3 install {package_name}",
+                f"python3 -m pip install {package_name}",
+                f"pip install {package_name}",
+            ]
+            
+            for command in install_commands:
+                result = self.shell_tools.run_shell_command(command, timeout=120)
+                
+                if "Successfully installed" in result or "already satisfied" in result:
+                    self.installed_packages.add(package_name)
+                    return f"Package '{package_name}' installed successfully"
+                
+                # If first method fails, try the next one
+                if "error" not in result.lower():
+                    break
+            
+            return f"Package installation failed: {result}"
+            
+        except Exception as e:
+            error_msg = f"Error installing package '{package_name}': {str(e)}"
+            logger.error(error_msg)
+            return error_msg
+    
+    def save_python_file(self, filename: str, code: str) -> str:
+        """
+        Save Python code to a file in the base directory.
+        
+        Args:
+            filename (str): Name of the Python file
+            code (str): Python code content
+            
+        Returns:
+            str: Success/failure message
+        """
+        try:
+            if not filename.endswith('.py'):
+                filename += '.py'
+            
+            filepath = self.base_dir / filename
+            
+            # Heal the code before saving
+            healed_code = self._heal_python_code(code)
+            
+            with open(filepath, 'w', encoding='utf-8') as f:
+                f.write(healed_code)
+            
+            logger.info(f"Python file saved: {filename}")
+            return f"Python file '{filename}' saved successfully to {self.base_dir}"
+            
+        except Exception as e:
+            error_msg = f"Error saving Python file '{filename}': {str(e)}"
+            logger.error(error_msg)
+            return error_msg
+    
+    def list_python_files(self) -> str:
+        """
+        List all Python files in the base directory.
+        
+        Returns:
+            str: List of Python files
+        """
+        try:
+            python_files = list(self.base_dir.glob("*.py"))
+            
+            if not python_files:
+                return "No Python files found in the base directory"
+            
+            file_list = []
+            for file_path in python_files:
+                file_stat = file_path.stat()
+                file_info = f"{file_path.name} ({file_stat.st_size} bytes, modified: {time.ctime(file_stat.st_mtime)})"
+                file_list.append(file_info)
+            
+            return "Python files in base directory:\n" + "\n".join(file_list)
+            
+        except Exception as e:
+            error_msg = f"Error listing Python files: {str(e)}"
+            logger.error(error_msg)
+            return error_msg
+    
+    def validate_python_syntax(self, code: str) -> str:
+        """
+        Validate Python code syntax without executing it.
+        
+        Args:
+            code (str): Python code to validate
+            
+        Returns:
+            str: Validation result message
+        """
+        try:
+            # Parse the code to check for syntax errors
+            ast.parse(code)
+            return "Python syntax is valid"
+            
+        except SyntaxError as e:
+            error_msg = f"Syntax Error at line {e.lineno}: {e.msg}"
+            logger.warning(f"Python syntax validation failed: {error_msg}")
+            return error_msg
+            
+        except Exception as e:
+            error_msg = f"Error validating Python syntax: {str(e)}"
+            logger.error(error_msg)
+            return error_msg
+    
+    def get_base_directory(self) -> str:
+        """
+        Get the current base directory path.
+        
+        Returns:
+            str: Absolute path of the base directory
+        """
+        return str(self.base_dir.absolute())
+    
+    def clear_temp_files(self) -> str:
+        """
+        Clean up any temporary Python files in the base directory.
+        
+        Returns:
+            str: Cleanup result message
+        """
+        try:
+            temp_files = list(self.base_dir.glob("temp_script_*.py"))
+            
+            if not temp_files:
+                return "No temporary files to clean up"
+            
+            for temp_file in temp_files:
+                temp_file.unlink()
+            
+            return f"Cleaned up {len(temp_files)} temporary Python files"
+            
+        except Exception as e:
+            error_msg = f"Error cleaning up temporary files: {str(e)}"
+            logger.error(error_msg)
+            return error_msg

utils/shell_toolkit.py

@@ -0,0 +1,137 @@
+"""
+Custom Shell Toolkit with Base Directory Support
+
+This toolkit provides shell command execution constrained to a specific base directory,
+preventing agents from navigating outside their assigned working directory.
+"""
+
+import os
+import subprocess
+from pathlib import Path
+from typing import List, Optional
+from agno.tools import Toolkit
+from agno.utils.log import logger
+
+
+class RestrictedShellTools(Toolkit):
+    """
+    Shell toolkit that restricts command execution to a specific base directory.
+    
+    This ensures agents cannot navigate outside their assigned working directory,
+    solving the issue of files being saved in wrong locations.
+    """
+    
+    def __init__(self, base_dir: Optional[Path] = None, **kwargs):
+        """
+        Initialize the restricted shell toolkit.
+        
+        Args:
+            base_dir: Base directory to constrain all shell operations to
+            **kwargs: Additional arguments passed to parent Toolkit
+        """
+        self.base_dir = Path(base_dir) if base_dir else Path.cwd()
+        
+        # Ensure base directory exists
+        self.base_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Initialize toolkit with our shell command function
+        super().__init__(
+            name="restricted_shell_tools", 
+            tools=[self.run_shell_command], 
+            **kwargs
+        )
+        
+        logger.info(f"RestrictedShellTools initialized with base_dir: {self.base_dir}")
+    
+    def run_shell_command(self, command: str, timeout: int = 30) -> str:
+        """
+        Runs a shell command in the constrained base directory.
+        
+        Args:
+            command (str): The shell command to execute
+            timeout (int): Maximum execution time in seconds
+            
+        Returns:
+            str: The output of the command or error message
+        """
+        try:
+            # Log the command and working directory
+            logger.info(f"Executing shell command in {self.base_dir}: {command}")
+            
+            # Ensure we're working in the correct directory
+            original_cwd = os.getcwd()
+            
+            try:
+                # Change to base directory before executing command
+                os.chdir(self.base_dir)
+                
+                # Execute the command in the base directory
+                result = subprocess.run(
+                    command,
+                    shell=True,
+                    capture_output=True,
+                    text=True,
+                    timeout=timeout,
+                    cwd=str(self.base_dir)  # Explicitly set working directory
+                )
+                
+                # Log execution details
+                logger.debug(f"Command executed with return code: {result.returncode}")
+                
+                if result.returncode != 0:
+                    error_msg = f"Command failed with return code {result.returncode}\nSTDERR: {result.stderr}\nSTDOUT: {result.stdout}"
+                    logger.warning(error_msg)
+                    return error_msg
+                
+                # Return successful output
+                output = result.stdout.strip()
+                logger.debug(f"Command output: {output[:200]}{'...' if len(output) > 200 else ''}")
+                return output
+                
+            finally:
+                # Always restore original working directory
+                os.chdir(original_cwd)
+                
+        except subprocess.TimeoutExpired:
+            error_msg = f"Command timed out after {timeout} seconds: {command}"
+            logger.error(error_msg)
+            return error_msg
+            
+        except Exception as e:
+            error_msg = f"Error executing command '{command}': {str(e)}"
+            logger.error(error_msg)
+            return error_msg
+    
+    def get_current_directory(self) -> str:
+        """
+        Returns the current base directory path.
+        
+        Returns:
+            str: Absolute path of the base directory
+        """
+        return str(self.base_dir.absolute())
+    
+    def list_directory_contents(self) -> str:
+        """
+        Lists the contents of the base directory.
+        
+        Returns:
+            str: Directory listing
+        """
+        return self.run_shell_command("ls -la")
+    
+    def check_file_exists(self, filename: str) -> str:
+        """
+        Checks if a file exists in the base directory.
+        
+        Args:
+            filename (str): Name of the file to check
+            
+        Returns:
+            str: Result of the check
+        """
+        file_path = self.base_dir / filename
+        if file_path.exists():
+            return f"File '{filename}' exists in {self.base_dir}"
+        else:
+            return f"File '{filename}' does not exist in {self.base_dir}"

workflow/financial_workflow.py

@@ -1,360 +1,349 @@
 """
-Financial Document Analysis Workflow using Agno Workflows
-Clean, pure-python implementation with structured outputs to avoid JSON parsing issues
+Financial Document Analysis Workflow - Agno Workflow 2.0 Implementation (Fixed)
+
+This workflow processes financial documents through a multi-agent system using the new
+step-based architecture introduced in Agno Workflow 2.0:
+1. Data Extractor Agent: Extracts structured financial data
+2. Data Arrangement Function: Organizes data into Excel-ready format
+3. Code Generator Agent: Creates professional Excel reports
+
+Built according to Agno Workflow 2.0 standards with simple sequential execution.
 """
 
 import json
+import time
 from pathlib import Path
-from typing import Dict, List, Optional, Iterator
-from pydantic import BaseModel, Field
+from typing import Optional, Dict, Any
+from textwrap import dedent
+import os
 
-from agno.agent import Agent, RunResponse
-from agno.models.google import Gemini  
-from agno.media import File
+from agno.agent import Agent
+from agno.models.google import Gemini
 from agno.tools.file import FileTools
+from agno.tools.shell import ShellTools
 from agno.tools.python import PythonTools
-from agno.workflow import Workflow
+from agno.workflow.v2.workflow import Workflow
+from agno.workflow.v2.types import StepInput, StepOutput
+from agno.workflow.v2.step import Step
+from agno.storage.sqlite import SqliteStorage  # Added this import
 from agno.utils.log import logger
-from agno.tools.shell import ShellTools
+from pydantic import BaseModel, Field
+
 from config.settings import settings
 from utils.prompt_loader import prompt_loader
+from utils.shell_toolkit import RestrictedShellTools
+from utils.restricted_python_tools import RestrictedPythonTools
 
 
-# Structured Output Models to avoid JSON parsing issues
 class DataPoint(BaseModel):
-    """Individual financial data point"""
-    field_name: str = Field(..., description="Name of the financial data field")
-    value: str = Field(..., description="Value of the field")
-    category: str = Field(..., description="Financial category (revenue, expenses, assets, etc.)")
+    """Individual financial data point."""
+    field_name: str = Field(description="Name of the financial data field")
+    value: str = Field(description="Value of the field")
+    category: str = Field(description="Financial category (revenue, expenses, assets, etc.)")
     period: str = Field(default="", description="Time period if applicable")
     unit: str = Field(default="", description="Currency or measurement unit")
     confidence: float = Field(default=0.9, description="Confidence score 0-1")
 
-class ExtractedFinancialData(BaseModel):
-    """Structured output for data extraction phase"""
-    company_name: str = Field(default="", description="Company name")
-    document_type: str = Field(..., description="Type of financial document")
-    reporting_period: str = Field(default="", description="Reporting period")
-    data_points: List[DataPoint] = Field(..., description="All extracted financial data points")
-    summary: str = Field(..., description="Brief summary of extracted data")
 
-class FinancialCategory(BaseModel):
-    """A category of organized financial data"""
-    category_name: str = Field(..., description="Name of the financial category")
-    description: str = Field(..., description="Description of what this category contains")
-    data_items: Dict[str, str] = Field(..., description="Key-value pairs of financial data")
-    totals: Dict[str, str] = Field(default_factory=dict, description="Any calculated totals")
+class Metadata(BaseModel):
+    """Metadata for extracted financial data."""
+    company_name: str = Field(default="Unknown Company", description="Company name")
+    document_type: str = Field(default="Unknown", description="Type of financial document")
+    reporting_period: str = Field(default="", description="Reporting period")
+    currency: str = Field(default="", description="Primary currency used")
 
-class ArrangedFinancialData(BaseModel):
-    """Structured output for data arrangement phase"""
-    categories: List[FinancialCategory] = Field(..., description="Organized financial categories")
-    key_metrics: Dict[str, str] = Field(default_factory=dict, description="Key financial metrics")
-    insights: List[str] = Field(default_factory=list, description="Financial insights and analysis")
-    summary: str = Field(..., description="Summary of arranged data")
 
-class GeneratedCode(BaseModel):
-    """Structured output for code generation phase"""
-    code: str = Field(..., description="Generated Python code for Excel creation")
-    description: str = Field(..., description="Description of what the code does")
-    output_filename: str = Field(..., description="Expected output filename")
-    execution_notes: str = Field(default="", description="Notes about code execution")
+class ExtractedFinancialData(BaseModel):
+    """Structured model for extracted financial data."""
+    data_points: list[DataPoint] = Field(description="List of extracted financial data points")
+    summary: str = Field(description="Summary of the extracted data")
+    metadata: Metadata = Field(default_factory=Metadata, description="Additional metadata")
 
 
 class FinancialDocumentWorkflow(Workflow):
     """
-    Pure Python workflow for financial document analysis
-    Uses structured outputs to eliminate JSON parsing issues
-    """
-    
-    description: str = "Financial document analysis workflow with data extraction, organization, and Excel generation"
+    Financial document analysis workflow using Agno Workflow 2.0 step-based architecture.
     
-    # Data Extractor Agent - Structured output eliminates JSON parsing issues
-    data_extractor: Agent = Agent(
-        model=Gemini(id=settings.DATA_EXTRACTOR_MODEL,thinking_budget=settings.DATA_EXTRACTOR_MODEL_THINKING_BUDGET,api_key=settings.GOOGLE_API_KEY),
-        description="Expert financial data extraction specialist",
-        instructions=prompt_loader.load_instructions_as_list("agents/data_extractor"),
-        response_model=ExtractedFinancialData,
-        structured_outputs=True,
-        debug_mode=True,
-    )
+    This workflow processes financial documents through three specialized steps:
+    - Data extraction with structured outputs
+    - Data arrangement for Excel compatibility  
+    - Excel report generation with formatting
+    """
     
-    # Data Arranger Agent - Organizes data into categories for Excel
-    data_arranger: Agent = Agent(
-        model=Gemini(id=settings.DATA_ARRANGER_MODEL,thinking_budget=settings.DATA_ARRANGER_MODEL_THINKING_BUDGET,api_key=settings.GOOGLE_API_KEY),
-        description="Financial data organization and analysis expert", 
-        instructions=prompt_loader.load_instructions_as_list("agents/data_arranger"),
-        tools=[FileTools()],  # FileTools for saving arranged data
-        # NOTE: Cannot use structured_outputs with tools in Gemini - choosing tools over structured outputs
-        markdown=True,
-        debug_mode=True,
-        add_memory_references=True,
-        add_session_summary_references=True,
-        exponential_backoff=True,
-        retries=10,
-    )
-
-    # Code Generator Agent - Creates Excel generation code
-    code_generator = Agent(
-        model=Gemini(
-            id=settings.CODE_GENERATOR_MODEL,
-            api_key=settings.GOOGLE_API_KEY
-        ),
-        description="Excel report generator that analyzes JSON data and creates formatted workbooks using shell execution on any OS",
-        goal="Generate a professional Excel report from arranged_financial_data.json with multiple worksheets, formatting, and charts",
-        instructions=prompt_loader.load_instructions_as_list("agents/code_generator"),
-        expected_output="A Financial_Report_YYYYMMDD_HHMMSS.xlsx file containing formatted data from the JSON with multiple worksheets, professional styling, and relevant charts",
-        additional_context="This agent must work on Windows, Mac, and Linux. Always use os.path for file operations and handle path separators correctly. Include proper error handling for cross-platform compatibility.",
-        tools=[
-            ShellTools(),
-            FileTools(save_files=True, read_files=True, list_files=True),
-            PythonTools(pip_install=True, save_and_run=False, run_code=False)
-        ],
-        markdown=False,
-        show_tool_calls=True,
-        debug_mode=True,
-        retries=10,
-        add_datetime_to_instructions=True,
-        delay_between_retries=10
-            )
-
-    def __init__(self, session_id: str = None, **kwargs):
-        super().__init__(session_id=session_id, **kwargs)
-        self.session_id = session_id or f"financial_workflow_{int(__import__('time').time())}"
-        self.session_output_dir = Path(settings.TEMP_DIR) / self.session_id / "output"
-        self.session_input_dir = Path(settings.TEMP_DIR) / self.session_id / "input"
-        self.session_temp_dir = Path(settings.TEMP_DIR) / self.session_id / "temp"
+    def __init__(self, session_id: Optional[str] = None, **kwargs):
+        """Initialize workflow with session management and step-based architecture."""
         
-        # Create all session directories
-        self.session_output_dir.mkdir(parents=True, exist_ok=True)
-        self.session_input_dir.mkdir(parents=True, exist_ok=True)  
-        self.session_temp_dir.mkdir(parents=True, exist_ok=True)
+        # Initialize session directories first
+        self._setup_session_directories(session_id)
         
-        # Configure tools with correct base directories after initialization
-        self._configure_agent_tools()
+        # Create storage with auto schema upgrade
+        storage = SqliteStorage(
+            table_name="financial_workflows",
+            db_file="tmp/agno_workflows.db",
+            mode="workflow_v2",  # Use workflow_v2 mode
+            auto_upgrade_schema=True  # This will fix your schema issues
+        )
         
-        logger.info(f"FinancialDocumentWorkflow initialized with session: {self.session_id}")
+        # Create agents for the workflow
+        self.data_extractor = self._create_data_extractor()
+        self.data_arranger = self._create_data_arranger()
+        self.code_generator = self._create_code_generator()
         
-    def clear_cache(self):
-        """Clear workflow session cache and temporary files."""
-        try:
-            # Clear session state
-            self.session_state.clear()
-            logger.info(f"Cleared workflow cache for session: {self.session_id}")
-            
-            # Clean up temporary files (keep input and output)
-            if self.session_temp_dir.exists():
-                import shutil
-                try:
-                    shutil.rmtree(self.session_temp_dir)
-                    self.session_temp_dir.mkdir(parents=True, exist_ok=True)
-                    logger.info(f"Cleaned temporary files for session: {self.session_id}")
-                except Exception as e:
-                    logger.warning(f"Could not clean temp directory: {e}")
-                    
-        except Exception as e:
-            logger.error(f"Error clearing workflow cache: {e}")
+        # Create steps using Step objects for better tracking
+        data_extraction_step = Step(
+            name="FinancialDataExtractor",
+            agent=self.data_extractor,
+            description="Expert financial data extraction specialist optimized for Gemini"
+        )
+        
+        data_arrangement_step = Step(
+            name="DataArrangement", 
+            executor=self._arrangement_function,
+            description="User-defined callable step for data arrangement"
+        )
+        
+        excel_generation_step = Step(
+            name="ExcelReportGenerator",
+            agent=self.code_generator,
+            description="Excel report generator optimized for Gemini with cross-platform support"
+        )
+        
+        # Initialize the Workflow 2.0 with step-based architecture
+        super().__init__(
+            name="FinancialDocumentWorkflow",
+            description=dedent("""\
+                Financial document analysis workflow using Agno Workflow 2.0 with step-based execution.
+                Processes financial documents through extraction, arrangement, and Excel report generation.
+                Uses session state for caching and proper error recovery mechanisms.
+            """),
+            steps=[
+                data_extraction_step,
+                data_arrangement_step,
+                excel_generation_step
+            ],
+            session_id=session_id,
+            storage=storage,  # Add the storage here
+            debug_mode=True,
+            **kwargs
+        )
+        
+        logger.info(f"FinancialDocumentWorkflow v2.0 initialized with session: {self.session_id}")
+        logger.info(f"Session directories: {list(self.session_directories.keys())}")
+    
+    def _setup_session_directories(self, session_id: Optional[str] = None):
+        """Setup session-specific directories."""
+        self.session_id = session_id
+        self.session_directories = settings.create_session_directories(self.session_id)
+        self.session_output_dir = self.session_directories["output"]
+        self.session_input_dir = self.session_directories["input"]
+        self.session_temp_dir = self.session_directories["temp"]
+        self.session_cache_dir = self.session_directories["cache"]
+    
+    def _create_data_extractor(self) -> Agent:
+        """Create the data extraction agent."""
+        return Agent(
+            model=Gemini(
+                id=settings.DATA_EXTRACTOR_MODEL,
+                thinking_budget=settings.DATA_EXTRACTOR_MODEL_THINKING_BUDGET,
+                api_key=settings.GOOGLE_API_KEY
+            ),
+            name="FinancialDataExtractor",
+            description="Expert financial data extraction specialist optimized for Gemini",
+            instructions=prompt_loader.load_instructions_as_list("agents/data_extractor"),
+            response_model=ExtractedFinancialData,
+            structured_outputs=True,
+            debug_mode=True,
+            retries=10,
+            delay_between_retries=10,
+            exponential_backoff=True,
+        )
     
-    def cleanup_session(self):
-        """Complete cleanup of session including all files."""
+    def _create_data_arranger(self) -> Agent:
+        """Create the data arrangement agent."""
+        logger.info(f"Data arranger base directory: {self.session_output_dir}")
+        logger.info(f"Directory exists: {self.session_output_dir.exists()}")
+        logger.info(f"Directory is writable: {os.access(self.session_output_dir, os.W_OK)}")
+        return Agent(
+            model=Gemini(
+                id=settings.DATA_ARRANGER_MODEL,
+                thinking_budget=settings.DATA_ARRANGER_MODEL_THINKING_BUDGET,
+                api_key=settings.GOOGLE_API_KEY
+            ),
+            name="FinancialDataArranger",
+            description="Financial data organization specialist optimized for Gemini",
+            instructions=prompt_loader.load_instructions_as_list("agents/data_arranger"),
+            tools=[
+                RestrictedShellTools(base_dir=self.session_output_dir),
+                FileTools(base_dir=self.session_output_dir, save_files=True, read_files=True, list_files=True),
+            ],
+            markdown=False,
+            debug_mode=True,
+            add_memory_references=True,
+            add_session_summary_references=True,
+            retries=10,
+            delay_between_retries=10,
+            exponential_backoff=True,
+        )
+    
+    def _create_code_generator(self) -> Agent:
+        """Create the code generation agent."""
+        return Agent(
+            model=Gemini(
+                id=settings.CODE_GENERATOR_MODEL,
+                thinking_budget=settings.CODE_GENERATOR_MODEL_THINKING_BUDGET,
+                api_key=settings.GOOGLE_API_KEY
+            ),
+            name="ExcelReportGenerator",
+            description="Excel report generator optimized for Gemini with cross-platform support",
+            goal="Generate professional Excel reports from arranged financial data with multiple worksheets and formatting",
+            instructions=prompt_loader.load_instructions_as_list("agents/code_generator"),
+            expected_output="A professionally formatted Excel file with multiple worksheets, charts, and proper styling",
+            additional_context=f"Working directory: {self.session_output_dir}. All files must be saved in this directory only.",
+            tools=[
+                RestrictedShellTools(base_dir=self.session_output_dir),
+                RestrictedPythonTools(base_dir=self.session_output_dir),
+                FileTools(base_dir=self.session_output_dir, save_files=True, read_files=True, list_files=True)
+            ],
+            markdown=False,
+            show_tool_calls=True,
+            debug_mode=True,
+            add_datetime_to_instructions=True,
+            retries=10,
+            delay_between_retries=10,
+            exponential_backoff=True,
+        )
+    
+    def _arrangement_function(self, step_input: StepInput) -> StepOutput:
+        """Custom function for data arrangement step."""
         try:
-            # Clear cache first
-            self.clear_cache()
+            message = step_input.message
+            previous_step_content = step_input.previous_step_content
+            
+            logger.info("Starting data arrangement step")
+            
+            # Load the base arrangement prompt
+            arrangement_prompt = prompt_loader.load_prompt("workflow/data_arrangement")
+            
+            # Combine prompt with extracted data from previous step
+            full_arrangement_prompt = f"{arrangement_prompt}\n\nHere is the extracted financial data to arrange:\n\n{previous_step_content}"
+            
+            # Run data arrangement using the agent
+            response = self.data_arranger.run(full_arrangement_prompt)
+            
+            # Cache the arrangement results in workflow session state
+            if hasattr(self, 'session_state') and self.session_state:
+                cache_key = f"arrangement_{int(time.time())}"
+                self.session_state[cache_key] = response.content
+                logger.info(f"Cached arrangement results with key: {cache_key}")
+            
+            logger.info("Data arrangement completed successfully")
+            
+            return StepOutput(
+                content=response.content,
+                response=response,
+                success=True
+            )
             
-            # Remove entire session directory
-            session_dir = Path(settings.TEMP_DIR) / self.session_id
-            if session_dir.exists():
-                import shutil
-                try:
-                    shutil.rmtree(session_dir)
-                    logger.info(f"Completely removed session directory: {session_dir}")
-                except Exception as e:
-                    logger.warning(f"Could not remove session directory: {e}")
-                    
         except Exception as e:
-            logger.error(f"Error during session cleanup: {e}")
-
-    def _configure_agent_tools(self):
-        """Configure agent tools with the correct base directories"""
-        # Configure data arranger's FileTools with session output directory
-        if hasattr(self.data_arranger, 'tools') and self.data_arranger.tools:
-            for tool in self.data_arranger.tools:
-                if isinstance(tool, FileTools):
-                    tool.base_dir = self.session_output_dir
-                    
-        # Configure code generator's tools with session output directory  
-        if hasattr(self.code_generator, 'tools') and self.code_generator.tools:
-            for tool in self.code_generator.tools:
-                if isinstance(tool, FileTools):
-                    tool.base_dir = self.session_output_dir
-                elif isinstance(tool, PythonTools):
-                    tool.base_dir = self.session_output_dir
-
-    def run(self, file_path: str = None, **kwargs) -> RunResponse:
+            logger.error(f"Data arrangement failed: {str(e)}")
+            return StepOutput(
+                content=f"Data arrangement failed: {str(e)}",
+                success=False,
+            )
+    
+    def run(self, file_path: str = None, **kwargs):
         """
-        Main workflow execution method
-        Pure Python workflow execution - no streaming, no JSON parsing issues
+        Main workflow execution using Workflow 2.0 step-based architecture.
+        
+        Args:
+            file_path: Path to the financial document to process
+            **kwargs: Additional parameters
+            
+        Returns:
+            Workflow execution result using the new step-based system
         """
-        # Handle file_path from parameter or attribute
+        # Handle file_path from kwargs if not provided as positional
         if file_path is None:
-            file_path = getattr(self, 'file_path', None)
+            file_path = kwargs.get('file_path')
         
         if file_path is None:
-            raise ValueError("file_path must be provided either as parameter or set as attribute")
-            
-        logger.info(f"Processing financial document: {file_path}")
+            logger.error("file_path is required but not provided")
+            raise ValueError("file_path is required but not provided")
         
-        # Remove use_cache parameter since it's not defined in the method signature
-        use_cache = kwargs.get('use_cache', True)
-        
-        # Check cache first if enabled
-        if use_cache and "final_results" in self.session_state:
-            logger.info("Returning cached results")
-            return RunResponse(
-                run_id=self.run_id,
-                content=self.session_state["final_results"]
-            )
+        start_time = time.time()
         
         try:
-            # Step 1: Extract Financial Data
-            logger.info("Step 1: Extracting financial data...")
+            # Validate input file
+            file_path = Path(file_path).resolve()
+            if not file_path.exists():
+                logger.error(f"File not found: {file_path}")
+                raise FileNotFoundError(f"File not found: {file_path}")
             
-            # Check for cached extraction
-            if use_cache and "extracted_data" in self.session_state:
-                extracted_data = ExtractedFinancialData.model_validate(
-                    self.session_state["extracted_data"]
-                )
-                logger.info("Using cached extraction data")
-            else:
-                document = File(filepath=file_path)
-                extraction_prompt = prompt_loader.load_prompt("workflow/data_extraction", file_path=file_path)
-                
-                extraction_response: RunResponse = self.data_extractor.run(
-                    extraction_prompt, 
-                    files=[document]
-                )
-                extracted_data: ExtractedFinancialData = extraction_response.content
-                
-                # Cache the result
-                self.session_state["extracted_data"] = extracted_data.model_dump()
-                logger.info(f"Extracted {len(extracted_data.data_points)} data points")
+            # Copy input file to session directory for reference
+            input_file = self.session_input_dir / file_path.name
+            input_file.write_bytes(file_path.read_bytes())
             
-            # Step 2: Arrange and Organize Data
-            logger.info("Step 2: Organizing financial data...")
+            logger.info(f"Starting financial document analysis for: {file_path.name}")
             
-            if use_cache and "arrangement_response" in self.session_state:
-                arrangement_content = self.session_state["arrangement_response"]
-                logger.info("Using cached arrangement data")
-            else:
-                # Debug: Check extracted data before passing to prompt
-                extracted_json = extracted_data.model_dump_json(indent=2)
-                logger.debug(f"Extracted data size: {len(extracted_json)} characters")
-                logger.debug(f"First 200 chars of extracted data: {extracted_json[:200]}...")
-                
-                arrangement_prompt = prompt_loader.load_prompt("workflow/data_arrangement", 
-                                                                extracted_data=extracted_json)
-                
-                # Debug: Check if prompt contains the actual data or just the placeholder
-                if "{extracted_data}" in arrangement_prompt:
-                    logger.error("CRITICAL: Variable substitution failed! Prompt still contains {extracted_data} placeholder")
-                    logger.error(f"Prompt length: {len(arrangement_prompt)}")
-                else:
-                    logger.info(f"Variable substitution successful. Prompt length: {len(arrangement_prompt)}")
-                
-                arrangement_response: RunResponse = self.data_arranger.run(arrangement_prompt)
-                arrangement_content = arrangement_response.content
-                
-                # Cache the result
-                self.session_state["arrangement_response"] = arrangement_content
-                logger.info("Data organization completed - check output directory for arranged_financial_data.json")
+            # Create File object for direct upload to Gemini API (for first step)
+            from agno.media import File
+            document = File(filepath=str(file_path))
             
-            # Step 3: Generate and Execute Excel Code
-            logger.info("Step 3: Generating and executing Excel code...")
-            
-            if use_cache and "code_generation_response" in self.session_state:
-                code_generation_content = self.session_state["code_generation_response"]
-                execution_success = self.session_state.get("execution_success", False)
-                logger.info("Using cached code generation results")
-            else:
-                code_prompt = prompt_loader.load_prompt("workflow/code_generation")
-                
-                code_response: RunResponse = self.code_generator.run(code_prompt)
-                code_generation_content = code_response.content
-                
-                # Simple check for execution success based on response content
-                execution_success = (
-                    "error" not in code_generation_content.lower() or
-                    "success" in code_generation_content.lower() or
-                    "completed" in code_generation_content.lower()
-                )
-                
-                # Cache the results
-                self.session_state["code_generation_response"] = code_generation_content
-                self.session_state["execution_success"] = execution_success
-                
-                logger.info(f"Code generation and execution completed: {'✅ Success' if execution_success else '❌ Failed'}")
+            # Load extraction prompt for the first step
+            extraction_prompt = prompt_loader.load_prompt(
+                "workflow/data_extraction",
+                file_path=str(file_path),
+                output_directory=str(self.session_output_dir)
+            )
             
-            # Prepare final results
-            # List actual output files
-            output_files = []
-            if self.session_output_dir.exists():
-                output_files = [f.name for f in self.session_output_dir.iterdir() if f.is_file()]
+            # Execute the workflow using the new 2.0 step-based system
+            # Pass the extraction prompt as the message and include the file
+            result = super().run(
+                message=extraction_prompt,
+                files=[document],
+                **kwargs
+            )
             
-            results_summary = f"""
-# Financial Document Analysis Complete
-
-## Document Information
-- **Company**: {extracted_data.company_name or 'Not specified'}
-- **Document Type**: {extracted_data.document_type}
-- **Reporting Period**: {extracted_data.reporting_period or 'Not specified'}
-
-## Processing Summary
-- **Data Points Extracted**: {len(extracted_data.data_points)}
-- **Data Organization**: {'✅ Completed' if arrangement_content else '❌ Failed'}
-- **Excel Creation**: {'✅ Success' if execution_success else '❌ Failed'}
-
-## Data Organization Results
-{arrangement_content[:500] + '...' if arrangement_content and len(arrangement_content) > 500 else arrangement_content or 'No arrangement data available'}
-
-## Tool Execution Summary
-**Data Arranger**: Used FileTools to save organized data to JSON
-**Code Generator**: Used PythonTools and FileTools for Excel generation
-
-## Code Generation Results  
-{code_generation_content[:500] + '...' if code_generation_content and len(code_generation_content) > 500 else code_generation_content or 'No code generation results available'}
-
-## Generated Files ({len(output_files)} files)
-{chr(10).join(f"- **{file}**" for file in output_files) if output_files else "- No files generated"}
-
-## Output Directory
-📁 `{self.session_output_dir}`
-
----
-*Generated using Agno Workflows with FileTools and PythonTools integration*
-*Note: Due to Gemini limitations, structured outputs were used for data extraction only*
-            """
+            # Final status
+            execution_time = time.time() - start_time
+            status = self._get_workflow_status()
             
-            # Cache final results
-            self.session_state["final_results"] = results_summary
+            logger.info(f"Workflow completed successfully in {execution_time:.2f} seconds")
+            logger.info(f"Results: {status}")
             
-            return RunResponse(
-                run_id=self.run_id,
-                content=results_summary
-            )
+            return result
             
         except Exception as e:
-            error_message = f"❌ Workflow failed: {str(e)}"
-            logger.error(f"Financial workflow error: {e}", exc_info=True)
-            return RunResponse(
-                run_id=self.run_id,
-                content=error_message
-            )
-
-    def get_processing_status(self) -> Dict[str, str]:
-        """Get the current processing status"""
+            logger.error(f"Workflow execution failed: {str(e)}")
+            raise
+    
+    def _get_workflow_status(self) -> Dict[str, Any]:
+        """Get current workflow status and file counts."""
         status = {
-            "extraction": "completed" if "extracted_data" in self.session_state else "pending",
-            "arrangement": "completed" if "arranged_data" in self.session_state else "pending", 
-            "code_generation": "completed" if "generated_code" in self.session_state else "pending",
-            "final_results": "completed" if "final_results" in self.session_state else "pending"
+            "session_id": self.session_id,
+            "output_directory": str(self.session_output_dir),
+            "json_files": 0,
+            "excel_files": 0,
+            "data_points": 0
         }
-        return status
+        
+        if self.session_output_dir.exists():
+            status["json_files"] = len(list(self.session_output_dir.glob("*.json")))
+            status["excel_files"] = len(list(self.session_output_dir.glob("*.xlsx")))
+        
+        return status
+
+
+# Compatibility function to maintain the same interface as the original workflow
+def create_financial_workflow(session_id: Optional[str] = None, **kwargs) -> FinancialDocumentWorkflow:
+    """
+    Create a new FinancialDocumentWorkflow instance using Workflow 2.0.
+    
+    Args:
+        session_id: Optional session ID for tracking workflow execution
+        **kwargs: Additional parameters for workflow configuration
+        
+    Returns:
+        FinancialDocumentWorkflow: Configured workflow instance
+    """
+    return FinancialDocumentWorkflow(session_id=session_id, **kwargs)

workflow/financial_workflow_working.py

@@ -0,0 +1,357 @@
+"""
+Financial Document Analysis Workflow - Optimized for Gemini Models
+
+This workflow processes financial documents through a multi-agent system:
+1. Data Extractor: Extracts structured financial data
+2. Data Arranger: Organizes data into Excel-ready format
+3. Code Generator: Creates professional Excel reports
+
+Built according to official Agno documentation standards.
+"""
+
+import json
+import time
+from pathlib import Path
+from typing import Iterator, Optional, Dict, Any
+from textwrap import dedent
+
+from agno.agent import Agent, RunResponse
+from agno.models.google import Gemini
+from agno.tools.file import FileTools
+from agno.tools.shell import ShellTools
+from agno.tools.python import PythonTools
+from agno.workflow import Workflow
+from agno.utils.log import logger
+from pydantic import BaseModel, Field
+
+from config.settings import settings
+from utils.prompt_loader import prompt_loader
+from utils.shell_toolkit import RestrictedShellTools
+from utils.restricted_python_tools import RestrictedPythonTools
+
+
+class DataPoint(BaseModel):
+    """Individual financial data point."""
+    field_name: str = Field(description="Name of the financial data field")
+    value: str = Field(description="Value of the field")
+    category: str = Field(description="Financial category (revenue, expenses, assets, etc.)")
+    period: str = Field(default="", description="Time period if applicable")
+    unit: str = Field(default="", description="Currency or measurement unit")
+    confidence: float = Field(default=0.9, description="Confidence score 0-1")
+
+
+class Metadata(BaseModel):
+    """Metadata for extracted financial data."""
+    company_name: str = Field(default="Unknown Company", description="Company name")
+    document_type: str = Field(default="Unknown", description="Type of financial document")
+    reporting_period: str = Field(default="", description="Reporting period")
+    currency: str = Field(default="", description="Primary currency used")
+
+
+class ExtractedFinancialData(BaseModel):
+    """Structured model for extracted financial data."""
+    data_points: list[DataPoint] = Field(description="List of extracted financial data points")
+    summary: str = Field(description="Summary of the extracted data")
+    metadata: Metadata = Field(default_factory=Metadata, description="Additional metadata")
+
+
+class FinancialDocumentWorkflow(Workflow):
+    """
+    Financial document analysis workflow optimized for Gemini models.
+    
+    This workflow processes financial documents through three specialized agents:
+    - Data extraction with structured outputs
+    - Data arrangement for Excel compatibility  
+    - Excel report generation with formatting
+    """
+    
+    description: str = dedent("""\
+        Financial document analysis workflow optimized for Gemini models with robust error handling.
+        Processes financial documents through extraction, arrangement, and Excel report generation.
+        Uses session state for caching and proper error recovery mechanisms.
+    """)
+    
+    # Data Extractor Agent - Uses structured outputs for reliable data extraction
+    data_extractor: Agent = Agent(
+        model=Gemini(
+            id=settings.DATA_EXTRACTOR_MODEL,
+            thinking_budget=settings.DATA_EXTRACTOR_MODEL_THINKING_BUDGET,
+            api_key=settings.GOOGLE_API_KEY
+        ),
+        name="FinancialDataExtractor",
+        description="Expert financial data extraction specialist optimized for Gemini",
+        instructions=prompt_loader.load_instructions_as_list("agents/data_extractor"),
+        response_model=ExtractedFinancialData,
+        structured_outputs=True,
+        debug_mode=True,
+        retries=10,
+        delay_between_retries=10,
+        exponential_backoff=True,
+    )
+    
+    def __init__(self, session_id: Optional[str] = None, **kwargs):
+        """Initialize workflow with session management."""
+        super().__init__(session_id=session_id, **kwargs)
+        
+        # Initialize session directories
+        self._setup_session_directories()
+        
+        # Initialize remaining agents with session-specific configurations
+        self._initialize_session_agents()
+        
+        logger.info(f"FinancialDocumentWorkflow initialized with session: {self.session_id}")
+        logger.info(f"Session directories: {list(self.session_directories.keys())}")
+    
+    def _setup_session_directories(self):
+        """Setup session-specific directories."""
+        self.session_directories = settings.create_session_directories(self.session_id)
+        self.session_output_dir = self.session_directories["output"]
+        self.session_input_dir = self.session_directories["input"]
+        self.session_temp_dir = self.session_directories["temp"]
+        self.session_cache_dir = self.session_directories["cache"]
+    
+    def _initialize_session_agents(self):
+        """Initialize agents that need session-specific configuration."""
+        
+        # Data Arranger Agent - Organizes data with file operations
+        self.data_arranger = Agent(
+            model=Gemini(
+                id=settings.DATA_ARRANGER_MODEL,
+                thinking_budget=settings.DATA_ARRANGER_MODEL_THINKING_BUDGET,
+                api_key=settings.GOOGLE_API_KEY
+            ),
+            name="FinancialDataArranger",
+            description="Financial data organization specialist optimized for Gemini",
+            instructions=prompt_loader.load_instructions_as_list("agents/data_arranger"),
+            tools=[
+                RestrictedShellTools(base_dir=self.session_output_dir),
+                FileTools(base_dir=self.session_output_dir)
+            ],
+            markdown=False,
+            debug_mode=True,
+            add_memory_references=True,
+            add_session_summary_references=True,
+            retries=10,
+            delay_between_retries=10,
+            exponential_backoff=True,
+            debug_level=2,
+        )
+
+        # Code Generator Agent - Creates Excel reports with comprehensive tools
+        self.code_generator = Agent(
+            model=Gemini(
+                id=settings.CODE_GENERATOR_MODEL,
+                thinking_budget=settings.CODE_GENERATOR_MODEL_THINKING_BUDGET,
+                api_key=settings.GOOGLE_API_KEY
+            ),
+            name="ExcelReportGenerator",
+            description="Excel report generator optimized for Gemini with cross-platform support",
+            goal="Generate professional Excel reports from arranged financial data with multiple worksheets and formatting",
+            instructions=prompt_loader.load_instructions_as_list("agents/code_generator"),
+            expected_output="A professionally formatted Excel file with multiple worksheets, charts, and proper styling",
+            additional_context=f"Working directory: {self.session_output_dir}. All files must be saved in this directory only.",
+            tools=[
+                RestrictedShellTools(base_dir=self.session_output_dir),
+                RestrictedPythonTools(base_dir=self.session_output_dir),
+                FileTools(base_dir=self.session_output_dir, save_files=True, read_files=True, list_files=True)
+            ],
+            markdown=False,
+            show_tool_calls=True,
+            debug_mode=True,
+            add_datetime_to_instructions=True,
+            retries=10,
+            delay_between_retries=10,
+            exponential_backoff=True,
+        )
+        
+        logger.info("All agents initialized with Gemini models and proper tool configuration")
+
+    def run(self, file_path: str = None, **kwargs) -> Iterator[RunResponse]:
+        """
+        Main workflow execution following official Agno documentation patterns.
+        
+        Args:
+            file_path: Path to the financial document to process
+            **kwargs: Additional parameters
+            
+        Yields:
+            RunResponse: Streaming responses from the workflow execution
+        """
+        # Handle file_path from kwargs if not provided as positional
+        if file_path is None:
+            file_path = kwargs.get('file_path')
+        
+        if file_path is None:
+            yield RunResponse(
+                run_id=self.run_id,
+                content="❌ Error: file_path is required but not provided"
+            )
+            return
+        
+        start_time = time.time()
+        
+        try:
+            # Validate input file
+            file_path = Path(file_path).resolve()
+            if not file_path.exists():
+                yield RunResponse(
+                    run_id=self.run_id,
+                    content=f"❌ Error: File not found: {file_path}"
+                )
+                return
+            
+            # Copy input file to session directory for reference
+            input_file = self.session_input_dir / file_path.name
+            input_file.write_bytes(file_path.read_bytes())
+            
+            yield RunResponse(
+                run_id=self.run_id,
+                content=f"🚀 Starting financial document analysis for: {file_path.name}"
+            )
+            
+            # Step 1: Data Extraction
+            yield RunResponse(
+                run_id=self.run_id,
+                content="📊 Step 1: Extracting financial data..."
+            )
+            
+            # Check cache first
+            cache_key = f"extraction_{file_path.name}_{file_path.stat().st_mtime}"
+            if cache_key in self.session_state:
+                logger.info("Using cached extraction results")
+                extracted_data = self.session_state[cache_key]
+                yield RunResponse(
+                    run_id=self.run_id,
+                    content="✅ Using cached extraction results"
+                )
+            else:
+                # Create File object for direct upload to Gemini API
+                from agno.media import File
+                document = File(filepath=str(file_path))
+                
+                # Load extraction prompt
+                extraction_prompt = prompt_loader.load_prompt(
+                    "workflow/data_extraction",
+                    file_path=str(file_path),
+                    output_directory=str(self.session_output_dir)
+                )
+                
+                # Run data extraction with file upload
+                extraction_response = self.data_extractor.run(extraction_prompt, files=[document])
+                extracted_data = extraction_response.content
+                
+                # Debug: Log the type of extracted_data
+                logger.info(f"DEBUG: extracted_data type: {type(extracted_data)}")
+                logger.info(f"DEBUG: extracted_data has model_dump_json: {hasattr(extracted_data, 'model_dump_json')}")
+                
+                # Cache the results
+                self.session_state[cache_key] = extracted_data
+                
+                yield RunResponse(
+                    run_id=self.run_id,
+                    content=f"✅ Extracted {len(extracted_data.data_points) if hasattr(extracted_data, 'data_points') else 'N/A'} data points"
+                )
+            
+            # Step 2: Data Arrangement
+            yield RunResponse(
+                run_id=self.run_id,
+                content="📋 Step 2: Arranging data for Excel..."
+            )
+            
+            # Load the base arrangement prompt (without placeholders)
+            arrangement_prompt = prompt_loader.load_prompt("workflow/data_arrangement")
+            
+            # Serialize extracted data
+            try:
+                if hasattr(extracted_data, 'model_dump_json'):
+                    extracted_data_json = extracted_data.model_dump_json(indent=2)
+                elif hasattr(extracted_data, 'model_dump'):
+                    import json
+                    extracted_data_json = json.dumps(extracted_data.model_dump(), indent=2)
+                else:
+                    import json
+                    extracted_data_json = json.dumps(str(extracted_data), indent=2)
+                
+                logger.info(f"DEBUG: Successfully serialized extracted_data ({len(extracted_data_json)} chars)")
+                
+            except Exception as e:
+                logger.error(f"DEBUG: Failed to serialize extracted_data: {e}")
+                import json
+                extracted_data_json = json.dumps({"error": "Failed to serialize extracted data", "data": str(extracted_data)}, indent=2)
+            
+            # Pass both the prompt and data directly to the agent
+            full_arrangement_prompt = f"{arrangement_prompt}\n\nHere is the extracted financial data to arrange:\n\n{extracted_data_json}"
+            
+            # Run data arrangement
+            arrangement_response = self.data_arranger.run(full_arrangement_prompt)
+            yield RunResponse(
+                run_id=self.run_id,
+                content=f"✅ Data arrangement completed"
+            )
+            
+            yield RunResponse(
+                run_id=self.run_id,
+                content="✅ Data arranged and saved to JSON"
+            )
+            
+            # Step 3: Excel Report Generation
+            yield RunResponse(
+                run_id=self.run_id,
+                content="📈 Step 3: Generating Excel report..."
+            )
+            
+            # Prepare code generation prompt
+            code_generation_prompt = prompt_loader.load_prompt(
+                "workflow/code_generation",
+                session_directory=str(self.session_output_dir)
+            )
+            
+            # Run code generation
+            code_generation_response = self.code_generator.run(code_generation_prompt)
+            yield RunResponse(
+                run_id=self.run_id,
+                content=f"✅ Excel report generation completed"
+            )
+            
+            # Final status
+            execution_time = time.time() - start_time
+            status = self._get_workflow_status()
+            
+            yield RunResponse(
+                run_id=self.run_id,
+                content=f"""
+✅ Workflow completed successfully in {execution_time:.2f} seconds
+
+📊 Results Summary:
+- Data points extracted: {status.get('data_points', 'N/A')}
+- JSON files created: {status.get('json_files', 0)}
+- Excel files created: {status.get('excel_files', 0)}
+- Session directory: {self.session_output_dir}
+
+🎯 All files saved to: {self.session_output_dir}
+                """.strip()
+            )
+            
+        except Exception as e:
+            logger.error(f"Workflow execution failed: {str(e)}")
+            yield RunResponse(
+                run_id=self.run_id,
+                content=f"❌ Workflow failed: {str(e)}"
+            )
+    
+    def _get_workflow_status(self) -> Dict[str, Any]:
+        """Get current workflow status and file counts."""
+        status = {
+            "session_id": self.session_id,
+            "output_directory": str(self.session_output_dir),
+            "json_files": 0,
+            "excel_files": 0,
+            "data_points": 0
+        }
+        
+        if self.session_output_dir.exists():
+            status["json_files"] = len(list(self.session_output_dir.glob("*.json")))
+            status["excel_files"] = len(list(self.session_output_dir.glob("*.xlsx")))
+        
+        return status
+