""" Reusable components for the Agent Leaderboard v2 These are stable components that don't change frequently """ def get_chart_colors(): return { "Private": "#1098F7", # Airglow Blue for Proprietary "Open source": "#58BC82", # Green for Open source "performance_bands": ["#DCFCE7", "#FEF9C3", "#FEE2E2"], "text": "#F5F6F7", "background": "#01091A", "grid": (0, 0, 0, 0.1), # RGBA tuple for grid } def get_rank_badge(rank): """Generate HTML for rank badge with appropriate styling""" badge_styles = { 1: ("1st", "linear-gradient(145deg, #ffd700, #ffc400)", "#000"), 2: ("2nd", "linear-gradient(145deg, #9ca3af, #787C7E)", "#fff"), 3: ("3rd", "linear-gradient(145deg, #CD7F32, #b36a1d)", "#fff"), } if rank in badge_styles: label, gradient, text_color = badge_styles[rank] return f"""
{label}
""" return f"""
{rank}
""" def get_type_badge(model_type): """Generate HTML for model type badge""" colors = get_chart_colors() colors = {"Private": colors["Private"], "Open source": colors["Open source"]} bg_color = colors.get(model_type, "#4F46E5") return f"""
{model_type}
""" def get_output_type_badge(output_type): """Generate HTML for output type badge""" if output_type == "Reasoning": bg_color = "#9333ea" # Purple for reasoning else: bg_color = "#6b7280" # Gray for normal return f"""
{output_type}
""" def get_score_bar(score): """Generate HTML for score bar with gradient styling and tooltip""" width = score * 100 return f"""
{score:.3f}
""" def get_metric_tooltip(metric): """Return tooltip text for different metrics""" tooltips = { "Avg AC": "Action Completion (AC): Measures how well the agent accomplishes user goals and completes tasks successfully. Higher is better (0-1 scale).", "Avg TSQ": "Tool Selection Quality (TSQ): Evaluates the accuracy of selecting the right tools and using them with correct parameters. Higher is better (0-1 scale).", "Avg Total Cost": "Average cost per conversation session in USD, including all API calls and processing. Lower is better.", "Avg Session Duration": "Average time taken to complete a full conversation session from start to finish, measured in seconds. Lower is generally better.", "Avg Turns": "Average number of back-and-forth exchanges needed to complete a task. Lower typically indicates more efficient task completion.", "Banking AC": "Action Completion score specific to banking domain tasks.", "Banking TSQ": "Tool Selection Quality score specific to banking domain tasks.", "Healthcare AC": "Action Completion score specific to healthcare domain tasks.", "Healthcare TSQ": "Tool Selection Quality score specific to healthcare domain tasks.", "Insurance AC": "Action Completion score specific to insurance domain tasks.", "Insurance TSQ": "Tool Selection Quality score specific to insurance domain tasks.", "Investment AC": "Action Completion score specific to investment domain tasks.", "Investment TSQ": "Tool Selection Quality score specific to investment domain tasks.", "Telecom AC": "Action Completion score specific to telecom domain tasks.", "Telecom TSQ": "Tool Selection Quality score specific to telecom domain tasks.", } return tooltips.get(metric, "") def get_responsive_styles(): """Return responsive CSS styles for mobile devices""" return """ """ def get_faq_section(): """Return the FAQ section HTML""" return """
❓

Frequently Asked Questions

Does the methodology favor GPT-4.1 since it uses GPT-4.1 to simulate users and tools, so GPT-4.1 ranks itself highest.
GPT's top ranking isn't due to simulator bias. Scenarios are pre-generated with Claude and fixed for all models. The user simulator drives goal-based conversations, and the tool simulator provides synthetic responses without influencing outcomes. Evaluation uses Claude as a judge, which should theoretically favor Claude (per sycophancy theory), but GPTs still lead.
Why does a specific model rank lower when our internal results show otherwise?
Performance varies by prompt, task, complexity, and domain. Our evaluations kept prompts identical across models for consistency. Different evaluation methodologies and task sets can lead to different rankings.
Why is my favorite model missing?
We were not able to add certain models either because they were not in our initial list or had issues while running the experiments, such as improper tool call output format. We skipped some of the models which performed poorly in our leaderboard v1.
We were surprised Gemini 2.5 Pro ranked lower. Our internal benchmarks show it's excellent for code research and AI code review tasks.
Results differ because this leaderboard evaluates support agent scenarios only, not coding ones. Different models excel at different types of tasks, and this benchmark focuses specifically on business support agent use cases across banking, healthcare, insurance, investment, and telecom domains.

📊 Understanding the Metrics

Action Completion (AC)

A score from 0 to 1 measuring how successfully the agent completes the user's requested tasks. This evaluates whether the agent achieves the intended goals, follows instructions accurately, and provides complete solutions. Higher scores indicate better task completion.

Tool Selection Quality (TSQ)

A score from 0 to 1 evaluating how well the agent selects and uses the appropriate tools for each task. This includes choosing the right tool, using correct parameters, and proper sequencing of tool calls. Higher scores indicate better tool utilization.

Domain-Specific Performance

Models are tested across five business domains: Banking, Healthcare, Insurance, Investment, and Telecom. Each domain has specific scenarios and requirements that test the agent's ability to handle industry-specific tasks and terminology.

Efficiency Metrics

• Cost: Total API cost per session in USD
• Duration: Time to complete tasks in seconds
• Turns: Number of exchanges to reach resolution
These metrics help identify the most cost-effective and efficient models for production use.

Learn More: For detailed methodology and evaluation criteria, visit the official blog post ↗ or explore the GitHub repository ↗

""" # Column mapping for sorting SORT_COLUMN_MAP = { "Avg Action Completion": "Avg AC", "Avg Tool Selection Quality": "Avg TSQ", "Avg Session Cost": "Avg Total Cost", }