New Leaderboard Update

- We now have the possibility to filter by task > filter by benchmarks
- We display the Aggregated Scores instead of the Average Scores

aggregated_scores.csv

@@ -0,0 +1,22 @@
+Model,Agg S2R,Agg MC,Agg VerilogEval S2R,Agg VerilogEval MC,Agg RTLLM,Agg VeriGen
+DeepSeek R1,74.84,75.51,77.01,77.81,68.06,54.4
+Llama 3.1 405B,49.72,42.8,53.98,42.92,36.43,41.67
+Llama 3.(1-3) 70B,39.0,38.49,38.64,37.45,40.12,48.05
+Qwen2.5 72B,49.23,48.82,49.17,51.22,49.45,26.75
+Qwen2.5 32B,50.58,40.73,50.53,41.85,50.71,30.46
+StarChat2 15B v0.1,39.04,38.9,37.45,37.69,44.0,49.99
+DeepSeek R1 Distill Qwen 14B,22.98,23.61,23.21,23.47,22.27,24.91
+CodeLlama 70B,31.46,31.29,34.17,29.8,22.99,44.96
+QwenCoder 2.5 32B,42.53,43.71,42.27,43.96,43.33,41.4
+DeepSeek Coder 33B,25.71,36.47,19.49,37.25,45.11,29.29
+QwenCoder 2.5 14B,36.75,38.49,35.61,39.03,40.33,33.55
+OpenCoder 8B,31.13,34.76,27.12,34.55,43.63,36.67
+QwenCoder 2.5 7B,13.86,32.31,6.31,31.75,37.41,37.47
+"DeepSeek Coder 6,7B",31.6,30.03,28.69,30.41,40.67,26.61
+RTLCoder Mistral,21.86,27.2,22.73,26.21,19.15,36.3
+RTLCoder DeepSeek,32.21,37.6,31.75,37.47,33.64,38.81
+OriGen,37.22,41.29,46.0,41.97,9.82,35.07
+HaVen-CodeQwen,41.66,46.09,42.97,46.57,37.55,41.74
+CodeV-CL-7B,28.19,35.7,25.75,35.39,35.79,38.53
+CodeV-QW-7B,20.79,47.26,18.73,50.28,27.23,19.55
+CodeV-DS-6.7B,18.19,44.1,14.28,47.05,30.39,17.03

app.py

@@ -3,8 +3,8 @@ import pandas as pd
 import gradio as gr
 from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 from css_html_js import custom_css, trigger_plot
-from parse import read_json, read_data
-from utils import model_hyperlink, filter_RTLRepo, filter_bench, filter_bench_all, handle_special_cases
+from parse import read_json, read_data, parse_agg
+from utils import model_hyperlink, filter_RTLRepo, filter_bench, filter_bench_all, handle_special_cases, type_emoji
 from typing import Union
 from about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT
 import numpy as np
@@ -12,12 +12,26 @@ import plotly.graph_objects as go
 import plotly.express as px
 from gradio.themes.utils import colors
 
-def filter_leaderboard(benchmark, model_type, search_query, max_params):
+def filter_leaderboard(task, benchmark, model_type, search_query, max_params):
     subset = df.copy()
+    
+    # Filter by task-specific benchmarks when 'All' benchmarks is selected
+    if task == "Spec-to-RTL":
+        valid_benchmarks = s2r_benchs
+        if benchmark == 'All':
+            subset = subset[subset['Benchmark'].isin(valid_benchmarks)]
+    elif task == "Code Completion":
+        valid_benchmarks = cc_benchs
+        if benchmark == 'All':
+            subset = subset[subset['Benchmark'].isin(valid_benchmarks)]
+    
+    # Apply benchmark filter if not "All"
     if benchmark != 'All':
         subset = df[df['Benchmark'] == benchmark]
+        
+    # Apply other filters
     if model_type != 'All':
-        model_type = model_type.split()[0]
+        # For dropdown without emojis
         subset = subset[subset['Model Type'] == model_type]
     if search_query:
         subset = subset[subset['Model'].str.contains(search_query, case=False, na=False)]
@@ -25,11 +39,33 @@ def filter_leaderboard(benchmark, model_type, search_query, max_params):
     subset = subset[subset['Params'] <= max_params]
 
     if benchmark == 'All':
-        return filter_bench_all(subset)
+        if task == 'Spec-to-RTL':
+            return filter_bench_all(subset, df_agg, agg_column='Agg S2R')
+        elif task == 'Code Completion':
+            return filter_bench_all(subset, df_agg, agg_column='Agg MC')
     elif benchmark == 'RTL-Repo':
         return filter_RTLRepo(subset)
     else:
-        return filter_bench(subset)
+        # Pass the specific benchmark aggregate column
+        agg_column = None
+        if benchmark == 'VerilogEval S2R':
+            agg_column = 'Agg VerilogEval S2R'
+        elif benchmark == 'VerilogEval MC':
+            agg_column = 'Agg VerilogEval MC'
+        elif benchmark == 'RTLLM':
+            agg_column = 'Agg RTLLM'
+        elif benchmark == 'VeriGen':
+            agg_column = 'Agg VeriGen'
+        
+        return filter_bench(subset, df_agg, agg_column)
+
+def update_benchmarks_by_task(task):
+    if task == "Spec-to-RTL":
+        return gr.update(choices=["All"] + s2r_benchs, value="All")
+    elif task == "Code Completion":
+        return gr.update(choices=["All"] + cc_benchs, value="All")
+    else:
+        return gr.update(choices=["All"] + benchmarks, value="All")
 
 def generate_scatter_plot(benchmark, metric):
     benchmark, metric = handle_special_cases(benchmark, metric)
@@ -50,7 +86,7 @@ def generate_scatter_plot(benchmark, metric):
     scatter_data['x'] = scatter_data['Params']
     scatter_data['y'] = scatter_data[metric]
     scatter_data['size'] = (scatter_data['x'] ** 0.3) * 40
-    
+
     type_colors = {"General": "green", "Coding": "yellow", "RTL-Specific": "blue"}
     scatter_data['color'] = scatter_data['Model Type'].map(type_colors).fillna('gray')
 
@@ -65,7 +101,10 @@ def generate_scatter_plot(benchmark, metric):
         scatter_data, x='x', y='y', log_x=True, size='size', color='Model Type', text='Model',
         hover_data={metric: ':.2f'}, title=f'Params vs. {metric} for {benchmark}',
         labels={'x': '# Params (Log Scale)', 'y': metric}, template="plotly_white",
-        # color_discrete_map={"General": "#A8D5BA", "Coding": "#F7DC6F", "RTL-Specific": "#87CEFA"},
+        # color_discrete_map={"General": "
+#A8D5BA", "Coding": "
+#F7DC6F", "RTL-Specific": "
+#87CEFA"},
         height=600, width=1200
     )
 
@@ -98,9 +137,14 @@ function refresh() {
         
 with gr.Blocks(css=custom_css, js=js_func, theme=gr.themes.Default(primary_hue=colors.emerald)) as app:
     df, benchmarks, metrics, default_metric = read_data()
+    df_agg = parse_agg("./aggregated_scores.csv")
+    tasks = ["Spec-to-RTL", "Code Completion"]
+    s2r_benchs = ["VerilogEval S2R", "RTLLM"]
+    cc_benchs = ["VerilogEval MC", "VeriGen", "RTL-Repo"]
     rtl_metrics = ["Exact Matching (EM)"]
-    non_rtl_metrics = ["Syntax (STX)", "Functionality (FNC)", "Synthesis (SYN)", "Power", "Performance", "Area"]
-    # gr.Markdown("""# TuRTLe 🐢 Model Leaderboard""")
+    non_rtl_metrics = ["Syntax (STX)", "Functionality (FNC)", "Synthesis (SYN)", "Power", "Performance", "Area", "Aggregated ⬆️"]
+    model_types = ['All', 'General', 'Coding', 'RTL-Specific']
+    
     gr.HTML("""
     <p align="center" style="margin-bottom: -10px;">
         <img src='/gradio_api/file=logo.png' alt='TuRTLe Logo' width='220'/> <br/>
@@ -135,22 +179,32 @@ with gr.Blocks(css=custom_css, js=js_func, theme=gr.themes.Default(primary_hue=c
     """)
     with gr.Tabs():
         with gr.Tab("Leaderboard"):
-            with gr.Row():
-                benchmark_radio = gr.Radio(choices=["All"] + benchmarks, label="Select Benchmark", value='VerilogEval S2R', scale=6)
-                model_type_radio = gr.Radio(choices=['All', 'General 🟢', 'Coding 🔵', 'RTL-Specific 🔴'], label="Select Model Type", value='All', scale=4)
+            with gr.Row(equal_height=True):
+                with gr.Column(scale=1):
+                    task_radio = gr.Radio(choices=tasks, label="Select Task", value='Spec-to-RTL')
+                with gr.Column(scale=1.75):
+                    benchmark_radio = gr.Radio(choices=["All"] + s2r_benchs, label="Select Benchmark", value='All')
             
-            with gr.Row():
-                search_box = gr.Textbox(label="Search Model", placeholder="Type model name...")
-                params_slider = gr.Slider(
-                    minimum=df['Params'].min(),
-                    maximum=700,
-                    value=700,
-                    label="Max Params",
-                    step=1
-                )
+            with gr.Row(equal_height=True):
+                with gr.Column(scale=1.9):
+                    search_box = gr.Textbox(label="Search Model", placeholder="Type model name...")
+                with gr.Column(scale=1):
+                    model_type_dropdown = gr.Dropdown(
+                        choices=model_types,
+                        label="Select Model Type",
+                        value='All'
+                    )
+                with gr.Column(scale=2):
+                    params_slider = gr.Slider(
+                        minimum=df['Params'].min(),
+                        maximum=700,
+                        value=700,
+                        label="Max Params",
+                        step=1
+                    )
             
             leaderboard = gr.DataFrame(
-                value=filter_leaderboard('VerilogEval S2R', 'All', "", 700),
+                value=filter_leaderboard('Spec-to-RTL', 'All', 'All', "", 700),
                 headers="first row",
                 show_row_numbers=True,
                 wrap=True,
@@ -159,9 +213,9 @@ with gr.Blocks(css=custom_css, js=js_func, theme=gr.themes.Default(primary_hue=c
                 column_widths=["7%", "25%", "10%", "17%", "6%", "6%", "6%", "6%", "6%", "7%"]),
         
         with gr.Tab("Interactive Bubble Plot"):
-            with gr.Row():
+            with gr.Row(equal_height=True):
                 bubble_benchmark = gr.Radio(choices=benchmarks, label="Select Benchmark", value='VerilogEval S2R')
-                bubble_metric = gr.Radio(choices=non_rtl_metrics, label="Select Metric", value="Syntax (STX)")
+                bubble_metric = gr.Radio(choices=non_rtl_metrics[:-1], label="Select Metric", value="Syntax (STX)")
             scatter_plot = gr.Plot(value=generate_scatter_plot('VerilogEval S2R', default_metric), label="Bubble Chart", elem_id="full-width-plot")
 
         with gr.Tab("About Us"):
@@ -202,22 +256,25 @@ with gr.Blocks(css=custom_css, js=js_func, theme=gr.themes.Default(primary_hue=c
                 )
     
     # event handlers, ugly way but it works
-    benchmark_radio.change(fn=filter_leaderboard, inputs=[benchmark_radio, model_type_radio, search_box, params_slider], outputs=leaderboard)
-    model_type_radio.change(fn=filter_leaderboard, inputs=[benchmark_radio, model_type_radio, search_box, params_slider], outputs=leaderboard)
-    search_box.change(fn=filter_leaderboard, inputs=[benchmark_radio, model_type_radio, search_box, params_slider], outputs=leaderboard)
-    params_slider.change(fn=filter_leaderboard, inputs=[benchmark_radio, model_type_radio, search_box, params_slider], outputs=leaderboard)
+    task_radio.change(
+        fn=update_benchmarks_by_task,
+        inputs=[task_radio],
+        outputs=[benchmark_radio]
+    )
+    task_radio.change(fn=filter_leaderboard, inputs=[task_radio, benchmark_radio, model_type_dropdown, search_box, params_slider], outputs=leaderboard)
+    benchmark_radio.change(fn=filter_leaderboard, inputs=[task_radio, benchmark_radio, model_type_dropdown, search_box, params_slider], outputs=leaderboard)
+    model_type_dropdown.change(fn=filter_leaderboard, inputs=[task_radio, benchmark_radio, model_type_dropdown, search_box, params_slider], outputs=leaderboard)
+    search_box.change(fn=filter_leaderboard, inputs=[task_radio, benchmark_radio, model_type_dropdown, search_box, params_slider], outputs=leaderboard)
+    params_slider.change(fn=filter_leaderboard, inputs=[task_radio, benchmark_radio, model_type_dropdown, search_box, params_slider], outputs=leaderboard)
 
-    # RTL-Repo Bubble plot handlres
+    # RTL-Repo Bubble plot
     def on_benchmark_change(benchmark, _):
         if benchmark == "RTL-Repo":
             metric = "Exact Matching (EM)"
             return gr.update(choices=rtl_metrics, value=metric), generate_scatter_plot(benchmark, metric)
         else:
             metric = non_rtl_metrics[0]  # default to Syntax
-            return gr.update(choices=non_rtl_metrics, value=metric), generate_scatter_plot(benchmark, metric)
-        # benchmark, metric = handle_special_cases(benchmark, metric)
-        # fig = generate_scatter_plot(benchmark, metric)
-        # return gr.update(value=metric), fig
+            return gr.update(choices=non_rtl_metrics[:-1], value=metric), generate_scatter_plot(benchmark, metric)
 
     def on_metric_change(benchmark, metric):
         benchmark, metric = handle_special_cases(benchmark, metric)

css_html_js.py

@@ -106,7 +106,10 @@ custom_css = """
     padding:0.5;
 }
 #box-filter > .form{
-    border: 0
+    border: 0;
+}
+.slider_input_container {
+    padding-top: 8px;
 }
 """
 

parse.py

@@ -30,14 +30,16 @@ model_details = {
     "OriGen": ("https://huggingface.co/henryen/OriGen_Fix", 6.74, "RTL-Specific")
 }
 
-def get_headers(reader) -> Union[list, list]:
+def get_headers(reader, agg=False) -> Union[list, list]:
     metrics, benchs = [], []
     for i, row in enumerate(reader):
         if i == 0:
             metrics = row[1:]
-        elif i == 1:
+        elif i == 1 and not agg:
             benchs = row[1:]
             break
+        else:
+            return metrics
     return metrics, benchs
 
 def get_model_params_and_url(model) -> Union[str, str, float]:
@@ -80,6 +82,13 @@ def parse_results(csv_path: str) -> list[dict]:
     print(models)
     return dataset
 
+def parse_agg(csv_path: str) -> list[dict]:
+    """
+    Each row has the following format:
+        MODEL | BENCHMARK | TASK | METRIC | RESULT
+    """
+    return pd.read_csv("aggregated_scores.csv")
+
 def writeJson(data: list):
     with open('results.json', 'w') as f:
         json.dump(data, f, indent=4, ensure_ascii=False)

utils.py

@@ -28,30 +28,51 @@ def filter_RTLRepo(subset: pd.DataFrame) -> pd.DataFrame:
     filtered_df['Type'] = filtered_df['Model Type'].map(lambda x: type_emoji.get(x, ""))
     filtered_df = filtered_df[['Type', 'Model', 'Params', 'Exact Matching (EM)']]
     filtered_df = filtered_df.sort_values(by='Exact Matching (EM)', ascending=False).reset_index(drop=True)
-    # filtered_df.insert(0, '', range(1, len(filtered_df) + 1))
     return filtered_df
 
-def filter_bench(subset: pd.DataFrame) -> pd.DataFrame:
+def filter_bench(subset: pd.DataFrame, df_agg=None, agg_column=None) -> pd.DataFrame:
     details = subset[['Model', 'Model URL', 'Model Type', 'Params']].drop_duplicates('Model')
     pivot_df = subset.pivot_table(index='Model', columns='Metric', values='Score', aggfunc='mean').reset_index()
-    pivot_df['Average ⬆️'] = pivot_df.mean(axis=1, numeric_only=True).round(2)
+    
+    if df_agg is not None and agg_column is not None and agg_column in df_agg.columns:
+        agg_data = df_agg[['Model', agg_column]].rename(columns={agg_column: 'Aggregated ⬆️'})
+        pivot_df = pd.merge(pivot_df, agg_data, on='Model', how='left')
+    else:# fallback
+        pivot_df['Aggregated ⬆️'] = pivot_df.mean(axis=1, numeric_only=True).round(2)
+    
     pivot_df = pd.merge(pivot_df, details, on='Model', how='left')
     pivot_df['Model'] = pivot_df.apply(lambda row: model_hyperlink(row["Model URL"], row["Model"]), axis=1)
     pivot_df['Type'] = pivot_df['Model Type'].map(lambda x: type_emoji.get(x, ""))
     pivot_df.rename(columns={'Syntax (STX)': 'STX', 'Functionality (FNC)': 'FNC', 'Synthesis (SYN)': 'SYN', 'Performance': 'Perf'}, inplace=True)
-    columns_order = ['Type', 'Model', 'Params', 'Average ⬆️', 'STX', 'FNC', 'SYN', 'Power', 'Perf', 'Area']
+    
+    columns_order = ['Type', 'Model', 'Params', 'Aggregated ⬆️', 'STX', 'FNC', 'SYN', 'Power', 'Perf', 'Area']
     pivot_df = pivot_df[[col for col in columns_order if col in pivot_df.columns]]
-    pivot_df = pivot_df.sort_values(by='Average ⬆️', ascending=False).reset_index(drop=True)
-    # pivot_df.insert(0, '', range(1, len(pivot_df) + 1))
+    pivot_df = pivot_df.sort_values(by='Aggregated ⬆️', ascending=False).reset_index(drop=True)
     return pivot_df
 
-def filter_bench_all(subset: pd.DataFrame) -> pd.DataFrame:
+def filter_bench_all(subset: pd.DataFrame, df_agg=None, agg_column=None) -> pd.DataFrame:
     details = subset[['Model', 'Model URL', 'Model Type', 'Params']].drop_duplicates('Model')
     pivot_df = subset.pivot_table(index='Model', columns='Metric', values='Score', aggfunc='mean').reset_index().round(2)
-    pivot_df['Average ⬆️'] = pivot_df.mean(axis=1, numeric_only=True).round(2)
+    
+    if df_agg is not None:
+        if agg_column is not None and agg_column in df_agg.columns:
+            agg_data = df_agg[['Model', agg_column]].rename(columns={agg_column: 'Aggregated ⬆️'})
+            pivot_df = pd.merge(pivot_df, agg_data, on='Model', how='left')
+        else:
+            agg_columns = [col for col in df_agg.columns if col.startswith('Agg ')]
+            if agg_columns:
+                df_agg['Average_Agg'] = df_agg[agg_columns].mean(axis=1)
+                agg_data = df_agg[['Model', 'Average_Agg']].rename(columns={'Average_Agg': 'Aggregated ⬆️'})
+                pivot_df = pd.merge(pivot_df, agg_data, on='Model', how='left')
+            else: # fallback
+                pivot_df['Aggregated ⬆️'] = pivot_df.mean(axis=1, numeric_only=True).round(2)
+    else: # fallback
+        pivot_df['Aggregated ⬆️'] = pivot_df.mean(axis=1, numeric_only=True).round(2)
+    
     pivot_df = pd.merge(pivot_df, details, on='Model', how='left')
     pivot_df['Model'] = pivot_df.apply(lambda row: model_hyperlink(row["Model URL"], row["Model"]), axis=1)
     pivot_df['Type'] = pivot_df['Model Type'].map(lambda x: type_emoji.get(x, ""))
+    
     pivot_df.rename(columns={
         'Exact Matching (EM)': 'EM', 
         'Syntax (STX)': 'Avg STX', 
@@ -61,9 +82,8 @@ def filter_bench_all(subset: pd.DataFrame) -> pd.DataFrame:
         'Performance': 'Avg Perf',
         'Area': 'Avg Area',
     }, inplace=True)
-    # columns_order = ['Type', 'Model', 'Params', 'Average ⬆️', 'Avg STX', 'Avg FNC', 'Avg SYN', 'Avg Power', 'Avg Perf', 'Avg Area']
-    columns_order = ['Type', 'Model', 'Params', 'Average ⬆️', 'Avg STX', 'Avg FNC', 'Avg SYN', 'Avg Power', 'Avg Perf', 'Avg Area']
+    
+    columns_order = ['Type', 'Model', 'Params', 'Aggregated ⬆️', 'Avg STX', 'Avg FNC', 'Avg SYN', 'Avg Power', 'Avg Perf', 'Avg Area']
     pivot_df = pivot_df[[col for col in columns_order if col in pivot_df.columns]]
-    pivot_df = pivot_df.sort_values(by='Average ⬆️', ascending=False).reset_index(drop=True)
-    # pivot_df.insert(0, '', range(1, len(pivot_df) + 1))
+    pivot_df = pivot_df.sort_values(by='Aggregated ⬆️', ascending=False).reset_index(drop=True)
     return pivot_df