Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
| """ | |
| Reusable components for the Agent Leaderboard v2 | |
| These are stable components that don't change frequently | |
| """ | |
| def get_chart_colors(): | |
| return { | |
| "Private": "#1098F7", # Airglow Blue for Proprietary | |
| "Open source": "#58BC82", # Green for Open source | |
| "performance_bands": ["#DCFCE7", "#FEF9C3", "#FEE2E2"], | |
| "text": "#F5F6F7", | |
| "background": "#01091A", | |
| "grid": (0, 0, 0, 0.1), # RGBA tuple for grid | |
| } | |
| def get_rank_badge(rank): | |
| """Generate HTML for rank badge with appropriate styling""" | |
| badge_styles = { | |
| 1: ("1st", "linear-gradient(145deg, #ffd700, #ffc400)", "#000"), | |
| 2: ("2nd", "linear-gradient(145deg, #9ca3af, #787C7E)", "#fff"), | |
| 3: ("3rd", "linear-gradient(145deg, #CD7F32, #b36a1d)", "#fff"), | |
| } | |
| if rank in badge_styles: | |
| label, gradient, text_color = badge_styles[rank] | |
| return f""" | |
| <div style=" | |
| display: inline-flex; | |
| align-items: center; | |
| justify-content: center; | |
| min-width: 48px; | |
| padding: 4px 12px; | |
| background: {gradient}; | |
| color: {text_color}; | |
| border-radius: 6px; | |
| font-weight: 600; | |
| font-size: 0.9em; | |
| box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2); | |
| "> | |
| {label} | |
| </div> | |
| """ | |
| return f""" | |
| <div style=" | |
| display: inline-flex; | |
| align-items: center; | |
| justify-content: center; | |
| min-width: 28px; | |
| color: #a1a1aa; | |
| font-weight: 500; | |
| "> | |
| {rank} | |
| </div> | |
| """ | |
| def get_type_badge(model_type): | |
| """Generate HTML for model type badge""" | |
| colors = get_chart_colors() | |
| colors = {"Private": colors["Private"], "Open source": colors["Open source"]} | |
| bg_color = colors.get(model_type, "#4F46E5") | |
| return f""" | |
| <div style=" | |
| display: inline-flex; | |
| align-items: center; | |
| padding: 4px 8px; | |
| background: {bg_color}; | |
| color: white; | |
| border-radius: 4px; | |
| font-size: 0.85em; | |
| font-weight: 500; | |
| "> | |
| {model_type} | |
| </div> | |
| """ | |
| def get_output_type_badge(output_type): | |
| """Generate HTML for output type badge""" | |
| if output_type == "Reasoning": | |
| bg_color = "#9333ea" # Purple for reasoning | |
| else: | |
| bg_color = "#6b7280" # Gray for normal | |
| return f""" | |
| <div style=" | |
| display: inline-flex; | |
| align-items: center; | |
| gap: 4px; | |
| padding: 4px 8px; | |
| background: {bg_color}; | |
| color: white; | |
| border-radius: 4px; | |
| font-size: 0.85em; | |
| font-weight: 500; | |
| "> | |
| {output_type} | |
| </div> | |
| """ | |
| def get_score_bar(score): | |
| """Generate HTML for score bar with gradient styling and tooltip""" | |
| width = score * 100 | |
| return f""" | |
| <div style="display: flex; align-items: center; gap: 12px; width: 100%;"> | |
| <div style=" | |
| flex-grow: 1; | |
| height: 8px; | |
| background: rgba(245, 246, 247, 0.1); | |
| border-radius: 4px; | |
| overflow: hidden; | |
| max-width: 200px; | |
| "> | |
| <div style=" | |
| width: {width}%; | |
| height: 100%; | |
| background: linear-gradient(90deg, #E35454, #1098F7); | |
| border-radius: 4px; | |
| transition: width 0.3s ease; | |
| "></div> | |
| </div> | |
| <span style=" | |
| font-family: 'SF Mono', monospace; | |
| font-weight: 600; | |
| color: #F5F6F7; | |
| min-width: 60px; | |
| ">{score:.3f}</span> | |
| </div> | |
| """ | |
| def get_metric_tooltip(metric): | |
| """Return tooltip text for different metrics""" | |
| tooltips = { | |
| "Avg AC": "Action Completion (AC): Measures how well the agent accomplishes user goals and completes tasks successfully. Higher is better (0-1 scale).", | |
| "Avg TSQ": "Tool Selection Quality (TSQ): Evaluates the accuracy of selecting the right tools and using them with correct parameters. Higher is better (0-1 scale).", | |
| "Avg Total Cost": "Average cost per conversation session in USD, including all API calls and processing. Lower is better.", | |
| "Avg Session Duration": "Average time taken to complete a full conversation session from start to finish, measured in seconds. Lower is generally better.", | |
| "Avg Turns": "Average number of back-and-forth exchanges needed to complete a task. Lower typically indicates more efficient task completion.", | |
| "Banking AC": "Action Completion score specific to banking domain tasks.", | |
| "Banking TSQ": "Tool Selection Quality score specific to banking domain tasks.", | |
| "Healthcare AC": "Action Completion score specific to healthcare domain tasks.", | |
| "Healthcare TSQ": "Tool Selection Quality score specific to healthcare domain tasks.", | |
| "Insurance AC": "Action Completion score specific to insurance domain tasks.", | |
| "Insurance TSQ": "Tool Selection Quality score specific to insurance domain tasks.", | |
| "Investment AC": "Action Completion score specific to investment domain tasks.", | |
| "Investment TSQ": "Tool Selection Quality score specific to investment domain tasks.", | |
| "Telecom AC": "Action Completion score specific to telecom domain tasks.", | |
| "Telecom TSQ": "Tool Selection Quality score specific to telecom domain tasks.", | |
| } | |
| return tooltips.get(metric, "") | |
| def get_responsive_styles(): | |
| """Return responsive CSS styles for mobile devices""" | |
| return """ | |
| <style> | |
| /* Enhanced mobile responsiveness */ | |
| @media (max-width: 768px) { | |
| /* Stack grid layouts vertically on mobile */ | |
| .insight-card-grid, | |
| .metric-card-grid { | |
| grid-template-columns: 1fr !important; | |
| gap: 12px !important; | |
| } | |
| /* Adjust table for mobile */ | |
| .v2-styled-table { | |
| font-size: 12px !important; | |
| } | |
| .v2-styled-table th, | |
| .v2-styled-table td { | |
| padding: 8px 6px !important; | |
| } | |
| /* Hide less important columns on mobile */ | |
| .v2-styled-table th:nth-child(8), | |
| .v2-styled-table td:nth-child(8), | |
| .v2-styled-table th:nth-child(9), | |
| .v2-styled-table td:nth-child(9) { | |
| display: none !important; | |
| } | |
| /* Make score bars more compact */ | |
| .score-cell { | |
| min-width: 120px !important; | |
| } | |
| /* Adjust domain selector for mobile */ | |
| .domain-radio .wrap { | |
| flex-direction: column !important; | |
| gap: 8px !important; | |
| } | |
| .domain-radio label { | |
| min-width: 100% !important; | |
| max-width: 100% !important; | |
| } | |
| /* Compact filter controls on mobile */ | |
| .compact-filter-row { | |
| flex-direction: column !important; | |
| } | |
| .compact-radio .wrap > label { | |
| font-size: 0.7rem !important; | |
| padding: 4px 8px !important; | |
| } | |
| /* Adjust navigation buttons */ | |
| .nav-buttons-container { | |
| flex-direction: column !important; | |
| gap: 8px !important; | |
| } | |
| .nav-link-button { | |
| width: 100% !important; | |
| justify-content: center !important; | |
| } | |
| /* Header adjustments */ | |
| h1 { | |
| font-size: 2rem !important; | |
| } | |
| h2 { | |
| font-size: 1.5rem !important; | |
| } | |
| h3 { | |
| font-size: 1.2rem !important; | |
| } | |
| /* Card padding adjustments */ | |
| .dark-container { | |
| padding: 16px !important; | |
| } | |
| .info-box { | |
| padding: 12px !important; | |
| } | |
| /* Chart container adjustments */ | |
| .chart-container { | |
| overflow-x: auto !important; | |
| -webkit-overflow-scrolling: touch !important; | |
| } | |
| /* Badge adjustments */ | |
| .badge-row { | |
| flex-wrap: wrap !important; | |
| } | |
| .badge { | |
| font-size: 0.65rem !important; | |
| padding: 3px 8px !important; | |
| } | |
| } | |
| @media (max-width: 480px) { | |
| /* Ultra-compact layout for very small screens */ | |
| .v2-styled-table { | |
| font-size: 10px !important; | |
| } | |
| /* Show only essential columns */ | |
| .v2-styled-table th:nth-child(n+6), | |
| .v2-styled-table td:nth-child(n+6) { | |
| display: none !important; | |
| } | |
| /* Keep only Rank, Model, Type, and main scores visible */ | |
| .v2-styled-table th:nth-child(5), | |
| .v2-styled-table td:nth-child(5) { | |
| display: table-cell !important; | |
| } | |
| /* Reduce all padding */ | |
| * { | |
| padding-left: 8px !important; | |
| padding-right: 8px !important; | |
| } | |
| /* Stack all buttons vertically */ | |
| .header-action-button { | |
| width: 90% !important; | |
| margin: 0 auto !important; | |
| } | |
| } | |
| /* Tooltip improvements for mobile */ | |
| @media (hover: none) and (pointer: coarse) { | |
| /* Show tooltips on tap for mobile */ | |
| .tooltip-trigger { | |
| position: relative; | |
| cursor: help; | |
| } | |
| .tooltip-trigger:active .tooltip-content, | |
| .tooltip-trigger:focus .tooltip-content { | |
| display: block !important; | |
| } | |
| } | |
| </style> | |
| """ | |
| def get_faq_section(): | |
| """Return the FAQ section HTML""" | |
| return """ | |
| <div class="dark-container" style="margin-top: 40px; margin-bottom: 40px;"> | |
| <div class="section-header"> | |
| <span class="section-icon" style="color: var(--accent-primary);">❓</span> | |
| <h3 style="margin: 0; color: var(--text-primary); font-size: 1.5rem; font-family: 'Geist', sans-serif; font-weight: 700;"> | |
| Frequently Asked Questions | |
| </h3> | |
| </div> | |
| <div style="margin-top: 24px;"> | |
| <!-- FAQ Item 1 --> | |
| <details class="faq-item" style="margin-bottom: 16px; background: var(--bg-secondary); border-radius: 12px; padding: 16px; border: 1px solid var(--border-subtle);"> | |
| <summary style="cursor: pointer; font-weight: 600; color: var(--text-primary); font-size: 1rem; display: flex; align-items: center; gap: 8px;"> | |
| <span style="color: var(--accent-primary);"></span> Does the methodology favor GPT-4.1 since it uses GPT-4.1 to simulate users and tools, so GPT-4.1 ranks itself highest. | |
| </summary> | |
| <div style="margin-top: 12px; padding-left: 28px; color: var(--text-secondary); line-height: 1.6;"> | |
| <strong style="color: var(--accent-secondary);"></strong> GPT's top ranking isn't due to simulator bias. Scenarios are pre-generated with Claude and fixed for all models. The user simulator drives goal-based conversations, and the tool simulator provides synthetic responses without influencing outcomes. Evaluation uses Claude as a judge, which should theoretically favor Claude (per sycophancy theory), but GPTs still lead. | |
| </div> | |
| </details> | |
| <!-- FAQ Item 2 --> | |
| <details class="faq-item" style="margin-bottom: 16px; background: var(--bg-secondary); border-radius: 12px; padding: 16px; border: 1px solid var(--border-subtle);"> | |
| <summary style="cursor: pointer; font-weight: 600; color: var(--text-primary); font-size: 1rem; display: flex; align-items: center; gap: 8px;"> | |
| <span style="color: var(--accent-primary);"></span> Why does a specific model rank lower when our internal results show otherwise? | |
| </summary> | |
| <div style="margin-top: 12px; padding-left: 28px; color: var(--text-secondary); line-height: 1.6;"> | |
| <strong style="color: var(--accent-secondary);"></strong> Performance varies by prompt, task, complexity, and domain. Our evaluations kept prompts identical across models for consistency. Different evaluation methodologies and task sets can lead to different rankings. | |
| </div> | |
| </details> | |
| <!-- FAQ Item 3 --> | |
| <details class="faq-item" style="margin-bottom: 16px; background: var(--bg-secondary); border-radius: 12px; padding: 16px; border: 1px solid var(--border-subtle);"> | |
| <summary style="cursor: pointer; font-weight: 600; color: var(--text-primary); font-size: 1rem; display: flex; align-items: center; gap: 8px;"> | |
| <span style="color: var(--accent-primary);"></span> Why is my favorite model missing? | |
| </summary> | |
| <div style="margin-top: 12px; padding-left: 28px; color: var(--text-secondary); line-height: 1.6;"> | |
| <strong style="color: var(--accent-secondary);"></strong> We were not able to add certain models either because they were not in our initial list or had issues while running the experiments, such as improper tool call output format. We skipped some of the models which performed poorly in our leaderboard v1. | |
| </div> | |
| </details> | |
| <!-- FAQ Item 4 --> | |
| <details class="faq-item" style="margin-bottom: 16px; background: var(--bg-secondary); border-radius: 12px; padding: 16px; border: 1px solid var(--border-subtle);"> | |
| <summary style="cursor: pointer; font-weight: 600; color: var(--text-primary); font-size: 1rem; display: flex; align-items: center; gap: 8px;"> | |
| <span style="color: var(--accent-primary);"></span> We were surprised Gemini 2.5 Pro ranked lower. Our internal benchmarks show it's excellent for code research and AI code review tasks. | |
| </summary> | |
| <div style="margin-top: 12px; padding-left: 28px; color: var(--text-secondary); line-height: 1.6;"> | |
| <strong style="color: var(--accent-secondary);"></strong> Results differ because this leaderboard evaluates support agent scenarios only, not coding ones. Different models excel at different types of tasks, and this benchmark focuses specifically on business support agent use cases across banking, healthcare, insurance, investment, and telecom domains. | |
| </div> | |
| </details> | |
| <!-- About Metrics --> | |
| <div style="margin-top: 32px; padding: 20px; background: linear-gradient(145deg, rgba(227, 84, 84, 0.05) 0%, rgba(16, 152, 247, 0.05) 100%); border-radius: 12px; border: 1px solid var(--border-default);"> | |
| <h4 style="color: var(--text-primary); margin-top: 0; margin-bottom: 16px; font-size: 1.2rem; font-family: 'Geist', sans-serif; font-weight: 600; display: flex; align-items: center; gap: 8px;"> | |
| <span style="font-size: 1.3rem;">📊</span> | |
| Understanding the Metrics | |
| </h4> | |
| <div style="display: grid; gap: 16px;"> | |
| <div> | |
| <h5 style="color: var(--accent-primary); margin: 0 0 8px 0; font-size: 1rem;">Action Completion (AC)</h5> | |
| <p style="color: var(--text-secondary); margin: 0; line-height: 1.5;"> | |
| A score from 0 to 1 measuring how successfully the agent completes the user's requested tasks. This evaluates whether the agent achieves the intended goals, follows instructions accurately, and provides complete solutions. Higher scores indicate better task completion. | |
| </p> | |
| </div> | |
| <div> | |
| <h5 style="color: var(--accent-primary); margin: 0 0 8px 0; font-size: 1rem;">Tool Selection Quality (TSQ)</h5> | |
| <p style="color: var(--text-secondary); margin: 0; line-height: 1.5;"> | |
| A score from 0 to 1 evaluating how well the agent selects and uses the appropriate tools for each task. This includes choosing the right tool, using correct parameters, and proper sequencing of tool calls. Higher scores indicate better tool utilization. | |
| </p> | |
| </div> | |
| <div> | |
| <h5 style="color: var(--accent-primary); margin: 0 0 8px 0; font-size: 1rem;">Domain-Specific Performance</h5> | |
| <p style="color: var(--text-secondary); margin: 0; line-height: 1.5;"> | |
| Models are tested across five business domains: Banking, Healthcare, Insurance, Investment, and Telecom. Each domain has specific scenarios and requirements that test the agent's ability to handle industry-specific tasks and terminology. | |
| </p> | |
| </div> | |
| <div> | |
| <h5 style="color: var(--accent-primary); margin: 0 0 8px 0; font-size: 1rem;">Efficiency Metrics</h5> | |
| <p style="color: var(--text-secondary); margin: 0; line-height: 1.5;"> | |
| • <strong>Cost:</strong> Total API cost per session in USD<br> | |
| • <strong>Duration:</strong> Time to complete tasks in seconds<br> | |
| • <strong>Turns:</strong> Number of exchanges to reach resolution<br> | |
| These metrics help identify the most cost-effective and efficient models for production use. | |
| </p> | |
| </div> | |
| </div> | |
| <div style="margin-top: 20px; padding-top: 16px; border-top: 1px solid var(--border-subtle);"> | |
| <p style="color: var(--text-secondary); margin: 0; font-size: 0.9rem; line-height: 1.5;"> | |
| <strong>Learn More:</strong> For detailed methodology and evaluation criteria, visit the | |
| <a href="https://galileo.ai/blog/agent-leaderboard-v2" target="_blank" style="color: var(--accent-primary); text-decoration: none;"> | |
| official blog post ↗ | |
| </a> | |
| or explore the | |
| <a href="https://github.com/rungalileo/agent-leaderboard" target="_blank" style="color: var(--accent-primary); text-decoration: none;"> | |
| GitHub repository ↗ | |
| </a> | |
| </p> | |
| </div> | |
| </div> | |
| </div> | |
| <style> | |
| .faq-item { | |
| transition: all 0.3s ease; | |
| } | |
| .faq-item:hover { | |
| border-color: var(--accent-primary) !important; | |
| box-shadow: 0 4px 12px rgba(227, 84, 84, 0.1); | |
| } | |
| .faq-item summary::-webkit-details-marker { | |
| display: none; | |
| } | |
| .faq-item summary::before { | |
| content: '▶'; | |
| display: inline-block; | |
| margin-right: 8px; | |
| transition: transform 0.3s ease; | |
| color: var(--accent-secondary); | |
| } | |
| .faq-item[open] summary::before { | |
| transform: rotate(90deg); | |
| } | |
| .faq-item summary:hover { | |
| color: var(--accent-primary) !important; | |
| } | |
| </style> | |
| </div> | |
| """ | |
| # Column mapping for sorting | |
| SORT_COLUMN_MAP = { | |
| "Avg Action Completion": "Avg AC", | |
| "Avg Tool Selection Quality": "Avg TSQ", | |
| "Avg Session Cost": "Avg Total Cost", | |
| } |