Update space

Makefile

@@ -1,13 +0,0 @@
-.PHONY: style format
-
-
-style:
-	python -m black --line-length 119 .
-	python -m isort .
-	ruff check --fix .
-
-
-quality:
-	python -m black --check --line-length 119 .
-	python -m isort --check-only .
-	ruff check .

README.md

@@ -1,5 +1,5 @@
 ---
-title: TuRTLe Leaderboard
+title: TuRTLe Model Leaderboard
 emoji: 🥇
 colorFrom: green
 colorTo: indigo
@@ -7,40 +7,6 @@ sdk: gradio
 app_file: app.py
 pinned: true
 license: apache-2.0
-short_description: 'TuRTLe: A Unified Evaluation of LLMs for RTL Generation'
+short_description: Duplicate this leaderboard to initialize your own!
 sdk_version: 5.19.0
 ---
-
-# Start the configuration
-
-Most of the variables to change for a default leaderboard are in `src/env.py` (replace the path for your leaderboard) and `src/about.py` (for tasks).
-
-Results files should have the following format and be stored as json files:
-```json
-{
-    "config": {
-        "model_dtype": "torch.float16", # or torch.bfloat16 or 8bit or 4bit
-        "model_name": "path of the model on the hub: org/model",
-        "model_sha": "revision on the hub",
-    },
-    "results": {
-        "task_name": {
-            "metric_name": score,
-        },
-        "task_name2": {
-            "metric_name": score,
-        }
-    }
-}
-```
-
-Request files are created automatically by this tool.
-
-If you encounter problem on the space, don't hesitate to restart it to remove the create eval-queue, eval-queue-bk, eval-results and eval-results-bk created folder.
-
-# Code logic for more complex edits
-
-You'll find 
-- the main table' columns names and properties in `src/display/utils.py`
-- the logic to read all results and request files, then convert them in dataframe lines, in `src/leaderboard/read_evals.py`, and `src/populate.py`
-- the logic to allow or filter submissions in `src/submission/submit.py` and `src/submission/check_validity.py`

about.py

@@ -0,0 +1,3 @@
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""
+"""

app.py

@@ -1,193 +1,122 @@
+import json
+import pandas as pd
 import gradio as gr
 from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+from css_html_js import custom_css
+from parse import read_json, read_data
+from utils import model_hyperlink, filter_RTLRepo, filter_bench, handle_special_cases
+from typing import Union
+from about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT
+import numpy as np
+import plotly.graph_objects as go
+import plotly.express as px
+
+def filter_leaderboard(benchmark, model_type, search_query, max_params):
+    subset = df[df['Benchmark'] == benchmark]
+    if model_type != 'All':
+        subset = subset[subset['Model Type'] == model_type]
+    if search_query:
+        subset = subset[subset['Model'].str.contains(search_query, case=False, na=False)]
+    max_params = float(max_params)
+    subset = subset[subset['Params'] <= max_params]
+    
+    if benchmark == 'RTL-Repo':
+        return filter_RTLRepo(subset)
+    else:
+        return filter_bench(subset)
+
+def generate_scatter_plot(benchmark, metric):
+    benchmark, metric = handle_special_cases(benchmark, metric)
+
+    subset = df[df['Benchmark'] == benchmark]
+    if benchmark == "RTL-Repo":
+        subset = subset[subset['Metric'].str.contains('EM', case=False, na=False)]
+        detailed_scores = subset.groupby('Model', as_index=False)['Score'].mean()
+        detailed_scores.rename(columns={'Score': 'EM'}, inplace=True)
+        detailed_scores['Average ⬆️'] = detailed_scores['EM']
+    else:
+        detailed_scores = subset.pivot_table(index='Model', columns='Metric', values='Score').reset_index()
+        detailed_scores['Average ⬆️'] = detailed_scores[['Syntax (STX)', 'Functionality (FNC)', 'Synthesis (SYN)', 'Power', 'Performance', 'Area']].mean(axis=1)
+
+    details = df[['Model', 'Params', 'Model Type']].drop_duplicates('Model')
+    scatter_data = pd.merge(detailed_scores, details, on='Model', how='left').dropna(subset=['Params', metric])
+
+    scatter_data['x'] = scatter_data['Params']
+    scatter_data['y'] = scatter_data[metric]
+    scatter_data['size'] = (scatter_data['x'] ** 0.3) * 40
+
+    type_colors = {"General": "green", "Coding": "yellow", "RTL-Specific": "blue"}
+    scatter_data['color'] = scatter_data['Model Type'].map(type_colors).fillna('gray')
+
+    y_axis_limits = {
+        'Functionality (FNC)': [5, 90], 'Syntax (STX)': [20, 100], 'Synthesis (SYN)': [5, 90],
+        'Power': [0, 50], 'Performance': [0, 50], 'Area': [0, 50], 'Exact Matching (EM)': [0, 50],
+        'Average ⬆️': [0, 80]
+    }
+    y_range = y_axis_limits.get(metric, [0, 80])
+
+    fig = px.scatter(
+        scatter_data, x='x', y='y', log_x=True, size='size', color='color', text='Model',
+        hover_data={metric: ':.2f'}, title=f'Params vs. {metric} for {benchmark}',
+        labels={'x': '# Params (Log Scale)', 'y': metric}, template="plotly_white",
+        height=600, width=1200
     )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+
+    fig.update_traces(
+        textposition='top center', textfont_size=10,
+        marker=dict(opacity=0.8, line=dict(width=0.5, color='black'))
     )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
+    fig.update_layout(
+        xaxis=dict(
+            showgrid=True, type='log', tickmode='array',
+            tickvals=[8, 14, 32, 72, 200, 700],
+            ticktext=['8', '14', '32', '72', '200', '700']
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
+        showlegend=False, yaxis=dict(range=y_range),
+        margin=dict(l=50, r=50, t=50, b=50), plot_bgcolor='white'
     )
 
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+    return fig
+
+with gr.Blocks(css=custom_css) as app:
+    df, benchmarks, metrics, default_metric = read_data()
+    gr.Markdown("""# TuRTLe 🐢 Model Leaderboard
+    Welcome to the TuRTLe Model Leaderboard! Use the filters below to explore different RTL benchmarks and models.
+    [GitHub Repository](https://github.com/https://github.com/HPAI-BSC) | [arXiv Preprint](https://arxiv.org/) | [How to submit](https://github.com/https://github.com/HPAI-BSC)<br/>
+    Contact us: hpai@bsc.es
+    """)
+    
+    with gr.Tabs():
+        with gr.Tab("Leaderboard"):
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
+                benchmark_radio = gr.Radio(choices=benchmarks, label="Select Benchmark", value='VerilogEval S2R', scale=7)
+                model_type_radio = gr.Radio(choices=['All', 'General', 'Coding', 'RTL-Specific'], label="Select Model Type", value='All', scale=4)
+            
             with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
-
+                search_box = gr.Textbox(label="Search Model", placeholder="Type model name...")
+                params_slider = gr.Slider(
+                    minimum=df['Params'].min(),
+                    maximum=700,
+                    value=700,
+                    label="Max Params",
+                    step=1
+                )
+            
+            leaderboard = gr.DataFrame(
+                value=filter_leaderboard('VerilogEval S2R', 'All', "", 700),
+                headers="first row",
+                wrap=True,
+                datatype=["markdown", "markdown", "html",],
+                interactive=False,
+                column_widths=["4%", "5%", "28%", "10%", "14%"],)
+        
+        with gr.Tab("Interactive Bubble Plot"):
+            with gr.Row():
+                bubble_benchmark = gr.Radio(choices=benchmarks, label="Select Benchmark", value='VerilogEval S2R')
+                bubble_metric = gr.Radio(choices=metrics, label="Select Metric", value=default_metric)
+            gr.Markdown("We show in 🟢 General Models, in 🟡 Coding Models and in 🔵 RTL-Specific Models. Detailed information is shown when hovering over each model in the plot.")
+            scatter_plot = gr.Plot(value=generate_scatter_plot('VerilogEval S2R', default_metric), label="Bubble Chart", elem_id="full-width-plot")
+    
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(
@@ -197,8 +126,35 @@ with demo:
                 elem_id="citation-button",
                 show_copy_button=True,
             )
+    
+    # event handlers, ugly way but it works
+    benchmark_radio.change(fn=filter_leaderboard, inputs=[benchmark_radio, model_type_radio, search_box, params_slider], outputs=leaderboard)
+    model_type_radio.change(fn=filter_leaderboard, inputs=[benchmark_radio, model_type_radio, search_box, params_slider], outputs=leaderboard)
+    search_box.change(fn=filter_leaderboard, inputs=[benchmark_radio, model_type_radio, search_box, params_slider], outputs=leaderboard)
+    params_slider.change(fn=filter_leaderboard, inputs=[benchmark_radio, model_type_radio, search_box, params_slider], outputs=leaderboard)
+
+    # RTL-Repo Bubble plot handlres
+    def on_benchmark_change(benchmark, metric):
+        benchmark, metric = handle_special_cases(benchmark, metric)
+        fig = generate_scatter_plot(benchmark, metric)
+        return gr.update(value=metric), fig
+
+    def on_metric_change(benchmark, metric):
+        benchmark, metric = handle_special_cases(benchmark, metric)
+        fig = generate_scatter_plot(benchmark, metric)
+        return gr.update(value=benchmark), fig
+
+    bubble_benchmark.change(
+        fn=on_benchmark_change, 
+        inputs=[bubble_benchmark, bubble_metric],
+        outputs=[bubble_metric, scatter_plot]
+    )
+
+    bubble_metric.change(
+        fn=on_metric_change,
+        inputs=[bubble_benchmark, bubble_metric],
+        outputs=[bubble_benchmark, scatter_plot]
+    )
+    
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+app.launch()

src/display/css_html_js.py → css_html_js.py

@@ -1,34 +1,43 @@
 custom_css = """
-
+#component-0 {
+    width: 75vw;
+    margin: 0 auto;
+    padding: 0px 40px;
+}
+@media (max-width: 1600px) {
+    #component-0 {
+        width: 85vw;
+        padding: 0px;
+    }
+}
+@media (max-width: 1100px) {
+    #component-0 {
+        width: 95vw;
+        padding: 0px;
+    }
+}
 .markdown-text {
     font-size: 16px !important;
 }
-
 #models-to-add-text {
     font-size: 18px !important;
 }
-
 #citation-button span {
     font-size: 16px !important;
 }
-
 #citation-button textarea {
     font-size: 16px !important;
 }
-
 #citation-button > label > button {
     margin: 6px;
     transform: scale(1.3);
 }
-
 #leaderboard-table {
     margin-top: 15px
 }
-
 #leaderboard-table-lite {
     margin-top: 15px
 }
-
 #search-bar-table-box > div:first-child {
     background: none;
     border: none;
@@ -37,7 +46,6 @@ custom_css = """
 #search-bar {
     padding: 0px;
 }
-
 /* Limit the width of the first AutoEvalColumn so that names don't expand too much */
 #leaderboard-table td:nth-child(2),
 #leaderboard-table th:nth-child(2) {
@@ -45,11 +53,9 @@ custom_css = """
     overflow: auto;
     white-space: nowrap;
 }
-
 .tab-buttons button {
     font-size: 20px;
 }
-
 #scale-logo {
     border-style: none !important;
     box-shadow: none;
@@ -58,7 +64,6 @@ custom_css = """
     margin-right: auto;
     max-width: 600px;
 }
-
 #scale-logo .download {
     display: none;
 }

main.py

@@ -0,0 +1,6 @@
+def main():
+    print("Hello from tortuga!")
+
+
+if __name__ == "__main__":
+    main()

parse.py

@@ -0,0 +1,106 @@
+import json
+import pandas as pd
+import csv
+from typing import Dict, Union
+import locale
+
+model_details = {
+    "DeepSeek R1": ("https://huggingface.co/deepseek-ai/DeepSeek-R1", 685, "General"),
+    "Llama 3.1 405B": ("https://huggingface.co/meta-llama/Llama-3.1-405B", 406, "General"),
+    "Llama 3.(1-3) 70B": ("https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct", 70.6, "General"),
+    "Qwen2.5 72B": ("https://huggingface.co/Qwen/Qwen2.5-72B-Instruct", 72.7, "General"),
+    "Qwen2.5 32B": ("https://huggingface.co/Qwen/Qwen2.5-32B", 32.5, "General"),
+    "StarChat2 15B v0.1": ("https://huggingface.co/HuggingFaceH4/starchat2-15b-v0.1", 16, "General"),
+    "DeepSeek R1 Distill Qwen 14B": ("https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", 14.8, "General"),
+    
+    "CodeLlama 70B": ("https://huggingface.co/codellama/CodeLlama-70b-hf", 69, "Coding"),
+    "QwenCoder 2.5 32B": ("https://huggingface.co/Qwen/Qwen2.5-Coder-32B-Instruct", 32.5, "Coding"),
+    "DeepSeek Coder 33B": ("https://huggingface.co/deepseek-ai/deepseek-coder-33b-instruct", 33.3, "Coding"),
+    "QwenCoder 2.5 14B": ("https://huggingface.co/Qwen/Qwen2.5-Coder-14B-Instruct", 14.7, "Coding"),
+    "OpenCoder 8B": ("https://huggingface.co/infly/OpenCoder-8B-Instruct", 7.77, "Coding"),
+    "QwenCoder 2.5 7B": ("https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct", 7.61, "Coding"),
+    "DeepSeek Coder 6,7B": ("https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct", 6.74, "Coding"),
+
+    "HaVen-CodeQwen": ("https://huggingface.co/yangyiyao/HaVen-CodeQwen", 7.25, "RTL-Specific"),
+    "CodeV-CL-7B": ("https://huggingface.co/yang-z/CodeV-CL-7B", 6.74, "RTL-Specific"),
+    "CodeV-QW-7B": ("https://huggingface.co/yang-z/CodeV-QW-7B", 7.25, "RTL-Specific"),
+    "CodeV-DS-6.7B": ("https://huggingface.co/yang-z/CodeV-DS-6.7B", 6.74, "RTL-Specific"),
+    "RTLCoder Mistral": ("https://huggingface.co/ishorn5/RTLCoder-v1.1", 7.24, "RTL-Specific"),
+    "RTLCoder DeepSeek": ("https://huggingface.co/ishorn5/RTLCoder-Deepseek-v1.1", 6.74, "RTL-Specific"),
+    "OriGen": ("https://huggingface.co/henryen/OriGen_Fix", 6.74, "RTL-Specific")
+}
+
+def get_headers(reader) -> Union[list, list]:
+    metrics, benchs = [], []
+    for i, row in enumerate(reader):
+        if i == 0:
+            metrics = row[1:]
+        elif i == 1:
+            benchs = row[1:]
+            break
+    return metrics, benchs
+
+def get_model_params_and_url(model) -> Union[str, str, float]:
+    if model not in model_details:
+        return "-", "-", "-"
+    url = model_details[model][0]
+    params = model_details[model][1]
+    type = model_details[model][2]
+    return url, params, type
+
+def parse_results(csv_path: str) -> list[dict]:
+    """
+    Each row has the following format:
+        MODEL | BENCHMARK | TASK | METRIC | RESULT
+    """
+    dataset = []
+    models = []
+    with open(csv_path, newline='') as csvfile:
+        reader = csv.reader(csvfile, delimiter=',')
+        metrics, benchs = get_headers(reader)
+        for i, row in enumerate(reader):
+            model = row[0]
+            url, params, type = get_model_params_and_url(model)
+            models.append(model)
+            row = row[1:]
+            ctr = 0
+            for metric, bench in zip(metrics, benchs):
+                record = {} 
+                record["Model"] = model
+                record["Model Type"] = type
+                record["Benchmark"] = bench
+                record["Task"] = metric
+                record["Result"] = float(row[ctr].replace(',','.'))
+                record["Model URL"] = url
+                record["Params"] = params
+                dataset.append(record)
+                ctr += 1
+    print(models)
+    return dataset
+
+def writeJson(data: list):
+    with open('results.json', 'w') as f:
+        json.dump(data, f, indent=4, ensure_ascii=False)
+    print("Done")
+
+def read_json():
+    json_path = "./results.json"
+    with open(json_path, "r", encoding="utf-8") as file:
+        data = json.load(file)
+    return data
+
+def read_data() -> Union[pd.DataFrame, list, list, str]:
+    data = read_json()
+    df = pd.DataFrame(data)
+    df.rename(columns={'Model': 'Model', 'Benchmark': 'Benchmark', 'Task': 'Metric', 'Result': 'Score'}, inplace=True)
+    df['Params'] = pd.to_numeric(df['Params'], errors='coerce')
+    benchmarks = sorted(df['Benchmark'].unique().tolist(), reverse=True)
+    metrics = df['Metric'].unique().tolist()
+    default_metric = 'Functionality (FNC)' if 'Functionality (FNC)' in metrics else metrics[0]
+    return df, benchmarks, metrics, default_metric
+
+
+if __name__ == "__main__":
+    csv_path = "./results.csv"
+    d = parse_results(csv_path)
+    writeJson(d)

pyproject.toml

@@ -1,13 +1,11 @@
-[tool.ruff]
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
-line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
-
-[tool.isort]
-profile = "black"
-line_length = 119
-
-[tool.black]
-line-length = 119
+[project]
+name = "tortuga"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "gradio>=5.21.0",
+    "gradio-leaderboard>=0.0.13",
+    "pandas>=2.2.3",
+]

requirements.txt

@@ -13,4 +13,5 @@ python-dateutil
 tqdm
 transformers
 tokenizers>=0.15.0
-sentencepiece
+sentencepiece
+plotly

results.csv

@@ -0,0 +1,23 @@
+,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area,EM,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area
+,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,RTL-Repo,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen
+DeepSeek R1,"96,54","91,43","79,74","67,76","78,97","63,27","38,94","35,64","37,82","31,95","38,76","34,5","33,02","97,95","94,12","80,26",60,"79,62","54,12","39,35","26,62","38,16","26,97","39,21","28,01"
+Llama 3.1 405B,"89,1","65,71","57,05","37,55","56,67","35,92","27,18","19,7",27,"16,04","26,79","18,91","33,29","91,41","72,94","44,74","45,88","44,1","44,71","21,98","19,24","20,74","22,19","21,66","21,08"
+Llama 3.(1-3) 70B,"67,69","73,88",40,"43,27","39,87","39,18","19,76","20,58","18,69","19,32","19,51","20,28","28,62","70,51","78,82","38,59","48,24","37,95","48,24","18,97","23,22","18,35","24,15","18,86","24,71"
+Qwen2.5 72B,"81,15","82,04","51,15","47,35","50,38","46,53","25,4","24,83","23,83","23,88","24,52","25,46","37,19","81,67","70,59","53,08","27,06","52,56","27,06","26,08","12,74","24,92","13,5","25,83","13,89"
+Qwen2.5 32B,"89,36","86,94","52,95","50,61","51,67","46,94","26,02","25,9","24,46","23,87","25,32","26,29","28,67","93,46","67,06","43,08","32,94","42,31","30,59","21,14",15,"20,48","15,38","21,15","15,31"
+StarChat2 15B v0.1,"86,54","85,71","38,72","42,45","38,59","42,45","19,18","22,44","17,99","21,03",19,"22,53","13,24","81,54","92,94","39,36","50,59","38,59","50,59","19,21","24,02","18,28","25,23","19,05","25,73"
+DeepSeek R1 Distill Qwen 14B,"41,28","40,82","23,85","20,82","23,72","20,41","11,81","12,46","11,25","10,24","11,76","10,7","20,65","42,18","62,35","24,36","25,88","24,1","25,88","11,96","11,54","11,38","12,93","11,86","12,9"
+CodeLlama 70B,"72,05","41,63","35,51","23,27","35,38","22,86","17,32","11,92","16,74","10,85","17,2","11,71","24,58","89,36","89,41","30,9","45,88","30,9","45,88","15,3","21,74","14,19","22,88","15,21","22,82"
+QwenCoder 2.5 32B,"87,69","79,59","45,64","43,27","43,33","42,04","21,51","22,02","20,72","20,95","21,17","22,03","30,44","84,87","72,94","45,51","41,18","44,87","41,18","22,26","20,56","21,48","20,67","22,2","20,87"
+DeepSeek Coder 33B,"57,82","83,67","19,87","43,67","19,87","42,86","9,94","23,28","9,83","21,19","9,47","23,2","30,58","78,72","83,53","39,49","29,41","38,33","29,41","18,92","14,52","18,2","14,74","18,76","14,67"
+QwenCoder 2.5 14B,"79,74","78,37","37,82","41,63","37,05","40,41","18,03","20,14","17,6","20,1","17,78","20,25","37,16","79,36","67,06","40,26","34,12","39,49","34,12","19,74","16,5","19,07","17,07","19,73","16,75"
+OpenCoder 8B,"75,77","75,1","28,59","46,53","28,21","42,86","13,81","22,24","13,16","21,47","13,71","21,73","16,63","79,87","92,94","36,03","43,53","35,51","37,65","17,57","17,19","16,74","18,76","17,52","19,06"
+QwenCoder 2.5 7B,"19,62","77,96","6,41","37,96","6,41","35,51","3,12","19,26","3,18","17,98","3,16","18,87","28,45","75,9","71,76","32,44","37,65","32,44","37,65","16,2","18,38","15,26","18,91","16,16","18,92"
+"DeepSeek Coder 6,7B","80,12","78,37","29,87","40,41","29,36","37,96","14,71","20,72","13,69","19,25","14,64","21,03","24,57","68,85","81,18","32,82","27,06","31,15","27,06","15,53","12,94","14,62","13,39","15,46","13,58"
+RTLCoder Mistral,"52,05","38,78","23,59","19,18","23,59","19,18","11,67","10,08","10,87","8,7","11,56","9,95","14,97","63,59","85,88","26,92","35,29","26,92","35,29","13,43","18,49","12,53","17,61","13,36","18,35"
+RTLCoder DeepSeek,"75,26","68,57","33,33","37,14","32,95","33,06","16,02","17,29","15,71","16,35","15,9","16,82","19,76","84,1","84,71","39,23","38,82","38,59","38,82","19,08","19,1","18,31","19,35","18,82","19,76"
+OriGen,"91,02","23,67","46,54","12,65","46,92","10,61","23,38","5,33","22,18","4,61","23,44","4,79","19,45","79,35","87,06","43,07","35,29","42,95","35,29","21,5","16,55","20,13","17,7","21,33","18,35"
+HaVen-CodeQwen,"90,26","82,45","45,9","40,41","44,36","38,37","21,77","19,1","21,23","18,31","21,46","18,92","25,38","93,33","97,65",50,"48,24","48,72","42,35","23,37","20,21","23,39","21,15","23,09","21,25"
+CodeV-CL-7B,"55,38","69,8","27,05","37,14","26,79","35,1","13,2","18,92","12,39","16,88","13,03","17,89","12,39","91,92","98,82","36,79","44,71","36,41","38,82","18,15","19,06","16,88","19,38","18,05","19,35"
+CodeV-QW-7B,"41,79","71,02","19,1","35,51","18,72","27,76","9,36","14,85","9,36","12,21","9,38","13,78","20,56","93,85","57,65","52,56","25,88","51,15",20,"25,64","9,39","24,22","9,99","25,56","9,94"
+CodeV-DS-6.7B,"30,77","62,45","14,87","33,88","14,62","30,61","7,3","15,49","6,9","14,75","7,22","15,35","21,06","95,13","58,82","48,85","23,53","48,33","17,65","24,02","8,26","22,82","8,81","23,73","8,47"

src/about.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-Intro text
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
-    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
-    link = f"https://huggingface.co/{model_name}"
-    return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
-    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
-    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,110 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/envs.py

@@ -1,25 +0,0 @@
-import os
-
-from huggingface_hub import HfApi
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
-
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,196 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/populate.py

@@ -1,58 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
-                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
-    """Checks if the model card and license exist and have been filled"""
-    try:
-        card = ModelCard.load(repo_id)
-    except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
-    # Enforce license metadata
-    if card.data.license is None:
-        if not ("license_name" in card.data and "license_link" in card.data):
-            return False, (
-                "License not found. Please add a license to your model card using the `license` metadata or a"
-                " `license_name`/`license_link` pair."
-            )
-
-    # Enforce card content
-    if len(card.text) < 200:
-        return False, "Please add a description to your model card, it is too short."
-
-    return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
-    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
-    try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-        if test_tokenizer:
-            try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
-            except ValueError as e:
-                return (
-                    False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
-                )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
-        return True, None, config
-
-    except ValueError:
-        return (
-            False,
-            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
-        )
-
-    except Exception as e:
-        return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size
-
-def get_model_arch(model_info: ModelInfo):
-    """Gets the model architecture from the configuration"""
-    return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
-    already_submitted_models,
-    check_model_card,
-    get_model_size,
-    is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    # Is the model on the hub?
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    # Is the model info correctly filled?
-    try:
-        model_info = API.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info, precision=precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-        "private": False,
-    }
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )

utils.py

@@ -0,0 +1,46 @@
+import pandas as pd
+import gradio as gr
+import plotly.graph_objects as go
+import plotly.express as px
+import numpy as np
+
+type_emoji = {
+    "RTL-Specific": "🔵",
+    "General": "🟢",
+    "Coding": "🟡"
+}
+
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+def handle_special_cases(benchmark, metric):
+    if metric == 'EM':
+        benchmark = 'RTL-Repo'
+    elif benchmark == 'RTL-Repo':
+        metric = 'EM'
+    return benchmark, metric
+
+def filter_RTLRepo(subset: pd.DataFrame) -> pd.DataFrame:
+    details = subset[['Model', 'Model URL', 'Model Type', 'Params']].drop_duplicates('Model')
+    filtered_df = subset[['Model', 'Score']].rename(columns={'Score': 'Exact Matching (EM)'})
+    filtered_df = pd.merge(filtered_df, details, on='Model', how='left')
+    filtered_df['Model'] = filtered_df.apply(lambda row: model_hyperlink(row["Model URL"], row["Model"]), axis=1)
+    filtered_df['Type'] = filtered_df['Model Type'].map(lambda x: type_emoji.get(x, ""))
+    filtered_df = filtered_df[['Type', 'Model', 'Params', 'Exact Matching (EM)']]
+    filtered_df.insert(0, '', range(1, len(filtered_df) + 1))
+    return filtered_df
+
+def filter_bench(subset: pd.DataFrame) -> pd.DataFrame:
+    details = subset[['Model', 'Model URL', 'Model Type', 'Params']].drop_duplicates('Model')
+    pivot_df = subset.pivot_table(index='Model', columns='Metric', values='Score', aggfunc='mean').reset_index()
+    pivot_df['Average ⬆️'] = pivot_df.mean(axis=1, numeric_only=True).round(2)
+    pivot_df = pd.merge(pivot_df, details, on='Model', how='left')
+    pivot_df['Model'] = pivot_df.apply(lambda row: model_hyperlink(row["Model URL"], row["Model"]), axis=1)
+    pivot_df['Type'] = pivot_df['Model Type'].map(lambda x: type_emoji.get(x, ""))
+    pivot_df.rename(columns={'Syntax (STX)': 'STX', 'Functionality (FNC)': 'FNC', 'Synthesis (SYN)': 'SYN', 'Performance': 'Perf'}, inplace=True)
+    columns_order = ['Type', 'Model', 'Params', 'Average ⬆️', 'STX', 'FNC', 'SYN', 'Power', 'Perf', 'Area']
+    pivot_df = pivot_df[[col for col in columns_order if col in pivot_df.columns]]
+    pivot_df = pivot_df.sort_values(by='Average ⬆️', ascending=False).reset_index(drop=True)
+    pivot_df.insert(0, '', range(1, len(pivot_df) + 1))
+    return pivot_df
+