Small changes

README.md

@@ -4,7 +4,7 @@ emoji: 🥇
 colorFrom: green
 colorTo: indigo
 sdk: gradio
-app_file: app.py
+app_file: example_app.py
 pinned: true
 license: apache-2.0
 short_description: Duplicate this leaderboard to initialize your own!

app.py

@@ -1,324 +1,120 @@
-import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
-    CITATION_BUTTON_LABEL,
-    CITATION_BUTTON_TEXT,
-    EVALUATION_QUEUE_TEXT,
-    INTRODUCTION_TEXT,
-    LLM_BENCHMARKS_TEXT,
-    TITLE,
-)
-
-from src.tasks import (
-    TE_DESCRIPTION,
-)
-
-from src.display.css_html_js import custom_css
-from src.display.utils import (
-    BENCHMARK_COLS,
-    COLS,
-    EVAL_COLS,
-    EVAL_TYPES,
-    AutoEvalColumn,
-    ModelType,
-    fields,
-    WeightType,
-    Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
-    API.restart_space(repo_id=REPO_ID)
-
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-def init_leaderboard(dataframe):
-    print(dataframe)
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
-
-
-def init_leaderboard2(dataframe, default_selection=None, hidden_columns=None):
-
-    print("entrato===============================================")
-
-    if dataframe is None or dataframe.empty:
-        raise ValueError("Leaderboard DataFrame is empty or None.")
-    return Leaderboard(
-        value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
-        select_columns=SelectColumns(
-            default_selection=default_selection or [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
-            label="Select Columns to Display:",
-        ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
-        hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden],
-        filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)"),
-            ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
-        ],
-        bool_checkboxgroup_label="Hide models",
-        interactive=False,
-    )
-
-
-demo = gr.Blocks(css=custom_css)
-with demo:
-    gr.HTML(TITLE)
-    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
-
-    with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 EVALITA-LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            #leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-            leaderboard = init_leaderboard2(
-                LEADERBOARD_DF,
-                default_selection=['T', 'Model', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"],
-                hidden_columns=[col for col in LEADERBOARD_DF.columns if
-                                col not in ['T', 'Model', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL" ]]
-            )
-
-
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
-                ],
-                submission_result,
-            )
-
-
-        with gr.TabItem("TE", elem_id="llm-benchmark-tab-table", id=4):
-            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
-            #leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-            LEADERBOARD_DF_TE = LEADERBOARD_DF.rename(columns={"TE Prompt Average": "Prompt Average",
-                                                            "TE Best Prompt": "Best Prompt",
-                                                            "TE Best Prompt Id": "Best Prompt Id",
-                                                            "TE": "Combined Performance"})
-
-            leaderboard = init_leaderboard2(
-                LEADERBOARD_DF_TE,
-                default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id'],
-                hidden_columns=[col for col in LEADERBOARD_DF.columns if
-                                col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id']]
-            )
-
-
-        with gr.TabItem("SA", elem_id="llm-benchmark-tab-table", id=5):
-            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
-
-            LEADERBOARD_DF_SA = LEADERBOARD_DF.rename(columns={"SA Prompt Average": "Prompt Average",
-                                                            "SA Best Prompt": "Best Prompt",
-                                                            "SA Best Prompt Id": "Best Prompt Id",
-                                                            "SA": "Combined Performance"})
-
-            leaderboard = init_leaderboard2(
-                LEADERBOARD_DF_SA,
-                default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
-                                   'Best Prompt Id'],
-                hidden_columns=[col for col in LEADERBOARD_DF.columns if
-                                col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
-                                            'Best Prompt Id']]
-            )
-
-
-
-
-        with gr.TabItem("HS", elem_id="llm-benchmark-tab-table", id=6):
-            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
-
-            LEADERBOARD_DF_HS = LEADERBOARD_DF.rename(columns={"HS Prompt Average": "Prompt Average",
-                                                               "HS Best Prompt": "Best Prompt",
-                                                               "HS Best Prompt Id": "Best Prompt Id",
-                                                               "HS": "Combined Performance"})
-
-            leaderboard = init_leaderboard2(
-                LEADERBOARD_DF_HS,
-                default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
-                                   'Best Prompt Id'],
-                hidden_columns=[col for col in LEADERBOARD_DF.columns if
-                                col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
-                                            'Best Prompt Id']]
-            )
-
-
-
-        with gr.TabItem("AT", elem_id="llm-benchmark-tab-table", id=7):
-            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
-
-        with gr.TabItem("WIC", elem_id="llm-benchmark-tab-table", id=8):
-            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
-
-        with gr.TabItem("FAQ", elem_id="llm-benchmark-tab-table", id=9):
-            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
-
-        with gr.TabItem("LS", elem_id="llm-benchmark-tab-table", id=10):
-            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
-
-        with gr.TabItem("SU", elem_id="llm-benchmark-tab-table", id=11):
-            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
-
-        with gr.TabItem("NER", elem_id="llm-benchmark-tab-table", id=12):
-            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
-
-        with gr.TabItem("REL", elem_id="llm-benchmark-tab-table", id=13):
-            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
-
-
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
+import gradio as gr
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE
+from src.tasks import TASK_DESCRIPTIONS, MEASURE_DESCRIPTION
+from src.display.css_html_js import custom_css
+from src.display.utils import BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn, ModelType, fields, WeightType, Precision
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval
+
+
+# Define task metadata (icons, names, descriptions)
+TASK_METADATA = {
+    "TE": {"icon": "📊", "name": "Textual Entailment", "tooltip": "Identify logical relationships between two text segments."},
+    "SA": {"icon": "😃", "name": "Sentiment Analysis", "tooltip": "Classify the sentiment (positive, negative, neutral) of a text."},
+    "HS": {"icon": "⚠️", "name": "Hate Speech", "tooltip": "Detect hate speech in a text."},
+    "AT": {"icon": "🏥", "name": "Admission Test", "tooltip": "Classify whether a clinical statement pertains to an admission test."},
+    "WIC": {"icon": "🔤", "name": "Word in Context", "tooltip": "Identify words in context and their meaning."},
+    "FAQ": {"icon": "❓", "name": "Frequently Asked Questions", "tooltip": "Answer frequently asked questions based on given text."},
+    "LS": {"icon": "🔄", "name": "Lexical Substitution", "tooltip": "Identify alternative words in a given context."},
+    "SU": {"icon": "📝", "name": "Summarization", "tooltip": "Summarize long text into a shorter version."},
+    "NER": {"icon": "🏷️", "name": "Named Entity Recognition", "tooltip": "Identify named entities (e.g., persons, locations, organizations) in text."},
+    "REL": {"icon": "🔗", "name": "Relation Extraction", "tooltip": "Extract and link laboratory test results to the respective tests in clinical narratives."},
+}
+
+def restart_space():
+    """Restart the Hugging Face space."""
+    API.restart_space(repo_id=REPO_ID)
+
+# Helper function for leaderboard initialization
+def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
+    """Initialize and return a leaderboard."""
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=default_selection or [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.fewshot_type.name, type="checkboxgroup", label="Few-Shot Learning (FS)"),
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)"),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
+
+def download_snapshot(repo, local_dir):
+    """Try to download a snapshot from Hugging Face Hub."""
+    try:
+        print(f"Downloading from {repo} to {local_dir}...")
+        snapshot_download(repo_id=repo, local_dir=local_dir, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN)
+    except Exception as e:
+        print(f"Error downloading {repo}: {e}")
+        restart_space()
+
+
+# Initialize the app by downloading snapshots
+download_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH)
+download_snapshot(RESULTS_REPO, EVAL_RESULTS_PATH)
+
+# Load leaderboard data
+LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+
+# Prepare the main interface
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        # Main leaderboard tab
+        with gr.TabItem("🏅 EVALITA-LLM Benchmark"):
+            leaderboard = init_leaderboard(
+                LEADERBOARD_DF,
+                default_selection=['FS', 'Model', "Avg. Combined Performance ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['FS', 'Model', "Avg. Combined Performance ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]]
+            )
+
+        # About tab
+        with gr.TabItem("📝 About"):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        # Task-specific leaderboards
+        for task, metadata in TASK_METADATA.items():
+            with gr.TabItem(f"{metadata['icon']}{task}"):
+
+                task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
+                gr.Markdown(task_description, elem_classes="markdown-text")
+
+                gr.Markdown(MEASURE_DESCRIPTION, elem_classes="markdown-text")
+
+                leaderboard = init_leaderboard(
+                    LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average", f"{task} Best Prompt": "Best Prompt", f"{task} Best Prompt Id": "Best Prompt Id", task: "Combined Performance"}),
+                    default_selection=['FS', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id'],
+                    hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['FS', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id']]
+                )
+
+    # Citation section
+    with gr.Accordion("📙 Citation", open=False):
+        gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=20, elem_id="citation-button", show_copy_button=True)
+
+# Background job to restart space
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+
+# Launch the app with concurrent queueing
 demo.queue(default_concurrency_limit=40).launch()

example_app.py

@@ -0,0 +1,324 @@
+import gradio as gr
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+
+from src.about import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    TITLE,
+)
+
+from src.tasks import (
+    TE_DESCRIPTION,
+)
+
+from src.display.css_html_js import custom_css
+from src.display.utils import (
+    BENCHMARK_COLS,
+    COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    AutoEvalColumn,
+    ModelType,
+    fields,
+    WeightType,
+    Precision
+)
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval
+
+
+def restart_space():
+    API.restart_space(repo_id=REPO_ID)
+
+### Space initialisation
+try:
+    print(EVAL_REQUESTS_PATH)
+    snapshot_download(
+        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
+try:
+    print(EVAL_RESULTS_PATH)
+    snapshot_download(
+        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+    )
+except Exception:
+    restart_space()
+
+
+LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+
+(
+    finished_eval_queue_df,
+    running_eval_queue_df,
+    pending_eval_queue_df,
+) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+
+def init_leaderboard(dataframe):
+    print(dataframe)
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            ColumnFilter(
+                AutoEvalColumn.params.name,
+                type="slider",
+                min=0.01,
+                max=150,
+                label="Select the number of parameters (B)",
+            ),
+            ColumnFilter(
+                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+            ),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
+
+def init_leaderboard2(dataframe, default_selection=None, hidden_columns=None):
+
+    print("entrato===============================================")
+
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    return Leaderboard(
+        value=dataframe,
+        datatype=[c.type for c in fields(AutoEvalColumn)],
+        select_columns=SelectColumns(
+            default_selection=default_selection or [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            label="Select Columns to Display:",
+        ),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)"),
+            ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
+        ],
+        bool_checkboxgroup_label="Hide models",
+        interactive=False,
+    )
+
+
+demo = gr.Blocks(css=custom_css)
+with demo:
+    gr.HTML(TITLE)
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🏅 EVALITA-LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+            #leaderboard = init_leaderboard(LEADERBOARD_DF)
+
+            leaderboard = init_leaderboard2(
+                LEADERBOARD_DF,
+                default_selection=['T', 'Model', "Average ⬆��", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if
+                                col not in ['T', 'Model', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL" ]]
+            )
+
+
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+            with gr.Column():
+                with gr.Row():
+                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+
+                with gr.Column():
+                    with gr.Accordion(
+                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            finished_eval_table = gr.components.Dataframe(
+                                value=finished_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
+                            )
+                    with gr.Accordion(
+                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            running_eval_table = gr.components.Dataframe(
+                                value=running_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
+                            )
+
+                    with gr.Accordion(
+                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                        open=False,
+                    ):
+                        with gr.Row():
+                            pending_eval_table = gr.components.Dataframe(
+                                value=pending_eval_queue_df,
+                                headers=EVAL_COLS,
+                                datatype=EVAL_TYPES,
+                                row_count=5,
+                            )
+            with gr.Row():
+                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+
+            with gr.Row():
+                with gr.Column():
+                    model_name_textbox = gr.Textbox(label="Model name")
+                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                    model_type = gr.Dropdown(
+                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                        label="Model type",
+                        multiselect=False,
+                        value=None,
+                        interactive=True,
+                    )
+
+                with gr.Column():
+                    precision = gr.Dropdown(
+                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                        label="Precision",
+                        multiselect=False,
+                        value="float16",
+                        interactive=True,
+                    )
+                    weight_type = gr.Dropdown(
+                        choices=[i.value.name for i in WeightType],
+                        label="Weights type",
+                        multiselect=False,
+                        value="Original",
+                        interactive=True,
+                    )
+                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+
+            submit_button = gr.Button("Submit Eval")
+            submission_result = gr.Markdown()
+            submit_button.click(
+                add_new_eval,
+                [
+                    model_name_textbox,
+                    base_model_name_textbox,
+                    revision_name_textbox,
+                    precision,
+                    weight_type,
+                    model_type,
+                ],
+                submission_result,
+            )
+
+
+        with gr.TabItem("TE", elem_id="llm-benchmark-tab-table", id=4):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+            #leaderboard = init_leaderboard(LEADERBOARD_DF)
+
+            LEADERBOARD_DF_TE = LEADERBOARD_DF.rename(columns={"TE Prompt Average": "Prompt Average",
+                                                            "TE Best Prompt": "Best Prompt",
+                                                            "TE Best Prompt Id": "Best Prompt Id",
+                                                            "TE": "Combined Performance"})
+
+            leaderboard = init_leaderboard2(
+                LEADERBOARD_DF_TE,
+                default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id'],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if
+                                col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id']]
+            )
+
+
+        with gr.TabItem("SA", elem_id="llm-benchmark-tab-table", id=5):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+            LEADERBOARD_DF_SA = LEADERBOARD_DF.rename(columns={"SA Prompt Average": "Prompt Average",
+                                                            "SA Best Prompt": "Best Prompt",
+                                                            "SA Best Prompt Id": "Best Prompt Id",
+                                                            "SA": "Combined Performance"})
+
+            leaderboard = init_leaderboard2(
+                LEADERBOARD_DF_SA,
+                default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
+                                   'Best Prompt Id'],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if
+                                col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
+                                            'Best Prompt Id']]
+            )
+
+
+
+
+        with gr.TabItem("HS", elem_id="llm-benchmark-tab-table", id=6):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+            LEADERBOARD_DF_HS = LEADERBOARD_DF.rename(columns={"HS Prompt Average": "Prompt Average",
+                                                               "HS Best Prompt": "Best Prompt",
+                                                               "HS Best Prompt Id": "Best Prompt Id",
+                                                               "HS": "Combined Performance"})
+
+            leaderboard = init_leaderboard2(
+                LEADERBOARD_DF_HS,
+                default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
+                                   'Best Prompt Id'],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if
+                                col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt',
+                                            'Best Prompt Id']]
+            )
+
+
+
+        with gr.TabItem("AT", elem_id="llm-benchmark-tab-table", id=7):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("WIC", elem_id="llm-benchmark-tab-table", id=8):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("FAQ", elem_id="llm-benchmark-tab-table", id=9):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("LS", elem_id="llm-benchmark-tab-table", id=10):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("SU", elem_id="llm-benchmark-tab-table", id=11):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("NER", elem_id="llm-benchmark-tab-table", id=12):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+        with gr.TabItem("REL", elem_id="llm-benchmark-tab-table", id=13):
+            gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+
+    with gr.Row():
+        with gr.Accordion("📙 Citation", open=False):
+            citation_button = gr.Textbox(
+                value=CITATION_BUTTON_TEXT,
+                label=CITATION_BUTTON_LABEL,
+                lines=20,
+                elem_id="citation-button",
+                show_copy_button=True,
+            )
+
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+demo.queue(default_concurrency_limit=40).launch()

app2.py → example_app2.py

@@ -8,7 +8,7 @@ from src.about import (
     CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT,
     INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE
 )
-from src.tasks import TE_DESCRIPTION
+from src.tasks import TASK_DESCRIPTIONS, MEASURE_DESCRIPTION
 from src.display.css_html_js import custom_css
 from src.display.utils import (
     BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn,
@@ -19,6 +19,53 @@ from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 
 
+
+
+# Define the task icons and names
+TASK_ICONS = {
+    "TE": "📊",  # Textual Entailment
+    "SA": "😃",  # Sentiment Analysis
+    "HS": "⚠️",  # Hate Speech
+    "AT": "🏥",  # Admission Test
+    "WIC": "🔤",  # Word in Context
+    "FAQ": "❓",  # Frequently Asked Questions
+    "LS": "🔄",  # Lexical Substitution
+    "SU": "📝",  # Summarization
+    "NER": "🏷️",  # Named Entity Recognition
+    "REL": "🔗",  # Relation Extraction
+}
+
+TASK_NAMES = {
+    "TE": "Textual Entailment",
+    "SA": "Sentiment Analysis",
+    "HS": "Hate Speech",
+    "AT": "Admission Test",
+    "WIC": "Word in Context",
+    "FAQ": "Frequently Asked Questions",
+    "LS": "Lexical Substitution",
+    "SU": "Summarization",
+    "NER": "Named Entity Recognition",
+    "REL": "Relation Extraction",
+}
+
+
+# Tooltip descriptions for each task
+TASK_TOOLTIPS = {
+    "TE": "Identify logical relationships between two text segments.",
+    "SA": "Classify the sentiment (positive, negative, neutral) of a text.",
+    "HS": "Detect hate speech in a text.",
+    "AT": "Classify whether a clinical statement pertains to an admission test.",
+    "WIC": "Identify words in context and their meaning.",
+    "FAQ": "Answer frequently asked questions based on given text.",
+    "LS": "Identify alternative words in a given context.",
+    "SU": "Summarize long text into a shorter version.",
+    "NER": "Identify named entities (e.g., persons, locations, organizations) in text.",
+    "REL": "Extract and link laboratory test results to the respective tests in clinical narratives.",
+}
+
+
+
+
 def restart_space():
     """Restart the Hugging Face space."""
     API.restart_space(repo_id=REPO_ID)
@@ -59,10 +106,11 @@ def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
         search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
         hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            #ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
+            ColumnFilter(AutoEvalColumn.fewshot_type.name, type="checkboxgroup", label="Few-Shot Learning (FS)"),
+            #ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
             ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)"),
-            ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
+            #ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
         ],
         bool_checkboxgroup_label="Hide models",
         interactive=False,
@@ -89,15 +137,16 @@ with demo:
         with gr.TabItem("🏅 EVALITA-LLM Benchmark", elem_id="llm-benchmark-tab-table"):
             leaderboard = init_leaderboard(
                 LEADERBOARD_DF,
-                default_selection=['T', 'Model', 'Few-Shot', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"],
+                default_selection=['FS', 'Model', "Avg. Combined Performance ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"],
                 hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in
-                                ['T', 'Model', 'Few-Shot', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]]
+                                ['FS', 'Model', "Avg. Combined Performance ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]]
             )
 
         # About tab
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table"):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
+        '''
         # Submission tab
         with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table"):
             gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
@@ -129,16 +178,30 @@ with demo:
                 [model_name_textbox, base_model_name_textbox, revision_name_textbox, precision, weight_type, model_type],
                 submission_result,
             )
+        '''
 
         # Task-specific leaderboards
         for task in ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]:
-            with gr.TabItem(task, elem_id="llm-benchmark-tab-table"):
-                gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text")
+
+            with gr.TabItem(f"{TASK_ICONS[task]}{task}", elem_id="llm-benchmark-tab-table"):
+
+                task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
+
+
+
+
+                gr.Markdown(task_description, elem_classes="markdown-text")
+
+
+                gr.Markdown(MEASURE_DESCRIPTION, elem_classes="markdown-text")
+
+
+
                 leaderboard = init_leaderboard(
                     prepare_leaderboard_df(LEADERBOARD_DF, task),
-                    default_selection=['T', 'Model', 'Few-Shot', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id'],
+                    default_selection=['FS', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id'],
                     hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in
-                                    ['T', 'Model', 'Few-Shot', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id']]
+                                    ['FS', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id']]
                 )
 
     # Citation section

get_model_info.py

@@ -35,20 +35,33 @@ for filename in os.listdir(input_folder):
             # Ottieni le informazioni del modello da Hugging Face
             model_info = api.model_info(model_name)
 
+            # Calcola il numero di parametri in miliardi, se disponibile
+            num_params = None
+            if model_info.safetensors and "BF16" in model_info.safetensors.parameters:
+                num_params = model_info.safetensors.parameters["BF16"] / 1e9  # Converti in miliardi
+
+            # Estrai la lingua (può essere una lista, quindi prendiamo la prima se esiste)
+            # Estrai e concatena i linguaggi
+            language = "_".join(model_info.card_data.get("language", [])) if model_info.card_data else ""
+
+            print(model_info)
+
             # Costruisci il dizionario con i metadati richiesti
             model_data = {
                 "model": model_name,
                 "base_model": model_info.config.get("architectures", [""])[0] if model_info.config else "",
                 "revision": model_info.sha,
-                "precision": "bfloat16",  # Se disponibile, sostituire con un valore reale
+                #"precision": "bfloat16",  # Se disponibile, sostituire con un valore reale
                 #"weight_type": "Original",
                 #"status": "FINISHED",
                 "submitted_time": str(model_info.created_at),
-                "model_type": "pretrained",
+                #"model_type": "pretrained",
                 #"likes": model_info.likes,
                 #"params": model_info.safetensors_size_in_bytes / 1e9 if model_info.safetensors_size_in_bytes else None,
                 #"license": model_info.license,
                 #"private": model_info.private,
+                "num_params_billion": num_params,  # Numero di parametri in miliardi
+                "language": language,  # Lingua estratta
             }
 
             # Separare il model_name in due parti: prima e dopo "/"
@@ -71,4 +84,4 @@ for filename in os.listdir(input_folder):
         except Exception as e:
             print(f"Error retrieving info for {model_name}: {e}")
 
-        print("Process completed.")
+        print("Process completed1.")

src/about.py

@@ -95,7 +95,9 @@ NUM_FEWSHOT = 0 # Change with your few shot
 
 # Your leaderboard name
 
-TITLE = """<h1 align="center" id="space-title">Work in progress!</h1>"""
+#TITLE = """<h1 align="center" id="space-title">Work in progress!</h1>"""
+# Your leaderboard name
+TITLE = """<h1 align="center" id="space-title">🚀 EVALITA-LLM Leaderboard 🚀</h1>"""
 
 
 # What does your leaderboard evaluate?

src/display/utils.py

@@ -23,18 +23,26 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+#auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+
+auto_eval_column_dict.append(["fewshot_type", ColumnContent, ColumnContent("FS", "str", True, never_hidden=True)])
+
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-auto_eval_column_dict.append(["fewshot", ColumnContent, ColumnContent("Few-Shot", "str", True)])
+#auto_eval_column_dict.append(["fewshot", ColumnContent, ColumnContent("Few-Shot", "str", True)])
+
+
+
+
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avg. Combined Performance ⬆️", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+#auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+#auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
 auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
@@ -50,7 +58,7 @@ class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
     private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
+    #precision = ColumnContent("precision", "str", True)
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
@@ -84,6 +92,34 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+
+
+@dataclass
+class FewShotDetails:
+    name: str
+    symbol: str = ""  # emoji
+
+class FewShotType(Enum):
+    ZS = FewShotDetails(name="zero-shot", symbol="0️⃣")
+    FS = FewShotDetails(name="5-few-shot", symbol="5️⃣")
+    Unknown = FewShotDetails(name="unknown", symbol="❓")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_num_fewshot(num_fewshot):
+        """Determines FewShotType based on num_fewshot."""
+        if num_fewshot == 0:
+            return FewShotType.ZS
+        if num_fewshot == 5:
+            return FewShotType.FS
+        return FewShotType.Unknown
+
+
+
+
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
@@ -133,4 +169,5 @@ new_column_dict.append(["ID", NewColumnContent, NewColumnContent("ID", "str", Tr
 NewColumn = make_dataclass("NewColumn", new_column_dict, frozen=True)
 
 # Includi questi nuovi valori nei COLS o in altre variabili di configurazione, se necessario
-NEW_COLS = [c.name for c in fields(NewColumn) if not c.hidden]
+NEW_COLS = [c.name for c in fields(NewColumn) if not c.hidden]
+

src/leaderboard/read_evals.py

@@ -1,259 +1,95 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    average_CPS: str
-    fewshot: str
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
-
-        average_CPS = data.get("average_CPS")
-
-        num_fewshot = config.get("num_fewshot", 0)  # Imposta il valore predefinito a 0
-        try:
-            num_fewshot = int(num_fewshot)  # Converte in intero se possibile
-        except ValueError:
-            num_fewshot = 0  # Se la conversione fallisce, assegna 0
-
-
-        precision = config.get("precision")
-
-        print(precision)
-
-        print(config, num_fewshot)
-
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        model_type = config.get("model_type")
-        # Modifica: Convertire model_type in un oggetto Enum (se è un Enum)
-        model_type = ModelType.from_str(model_type) if model_type else None
-
-        print("=====================", model_type, config.get("model_name"))
-
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        print(precision.value.name)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            #result_key = f"{model}_{precision.value.name}"
-            result_key = f"{model}_{num_fewshot}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            #result_key = f"{org}_{model}_{precision.value.name}"
-            result_key = f"{org}_{model}_{num_fewshot}"
-        full_model = "/".join(org_and_model)
-
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            '''
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-            '''
-
-            for k, v in data["tasks"].items():
-                #if task.benchmark == k:
-                if task.benchmark[:-2] == k:
-                    # print(k, "==================", v)
-                    # results[task.benchmark] = v[task.cps]
-
-                    #print(task.benchmark, v[task.metric])
-
-                    if "Best Prompt Id" in task.col_name:
-                        results[task.benchmark] = int(v[task.metric_type][-1:])
-                        #print(results[task.benchmark],v[task.metric_type][-1:])
-                    else:
-                        results[task.benchmark] = v[task.metric_type]
-
-
-                    #results[task.benchmark + "_" + task.metric] = 1.0
-
-
-                    #results[task.benchmark] = v[task.accuracy]
-                    # print("======", results[task.benchmark])
-                    #results[task.benchmark] = 1.0
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            average_CPS=average_CPS,
-            fewshot=num_fewshot,
-            model_type=model_type,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
-        )
-
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        #average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        average = self.average_CPS
-        fewshot = self.fewshot
-        print("?????", fewshot)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            #AutoEvalColumn.model_type.name: self.model_type.value.name,
-            #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-
-            AutoEvalColumn.model_type.name: self.model_type.value.name if self.model_type else "Unknown",
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol if self.model_type else "Unknown",
-
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.fewshot.name: fewshot,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        #eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict() # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, FewShotType
+from src.submission.check_validity import is_model_on_hub
+
+@dataclass
+class EvalResult:
+    eval_name: str
+    full_model: str
+    org: str
+    model: str
+    revision: str
+    results: dict
+    average_CPS: str
+    fewshot: int
+    fewshot_type: FewShotType = FewShotType.Unknown
+    weight_type: WeightType = WeightType.Original
+    architecture: str = "Unknown"
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = ""
+    still_on_hub: bool = False
+
+    @classmethod
+    def init_from_json_file(cls, json_filepath):
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+        average_CPS = f"{data.get('average_CPS'):.2f}"
+
+        num_fewshot = int(config.get("num_fewshot", 0))
+        fewshot_type = FewShotType.from_num_fewshot(num_fewshot)
+
+        model_type = ModelType.from_str(config.get("model_type")) if config.get("model_type") else None
+        num_params = math.ceil(config.get("num_params_billion", 0)) if config.get("num_params_billion") else 0
+
+        org_and_model = config.get("model_name", "").split("/", 1)
+        org, model = (org_and_model if len(org_and_model) == 2 else (None, org_and_model[0]))
+
+        full_model = "/".join([org, model] if org else [model])
+        still_on_hub, _, model_config = is_model_on_hub(full_model, config.get("model_sha", "main"))
+
+        architecture = ";".join(getattr(model_config, "architectures", [])) if model_config else "?"
+
+        results = {
+            task.value.benchmark: f"{data['tasks'].get(task.value.benchmark, {}).get(task.metric_type, 0):.2f}"
+            for task in Tasks
+        }
+
+        return cls(
+            eval_name=f"{model}_{num_fewshot}",
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            average_CPS=average_CPS,
+            fewshot=fewshot_type,
+            fewshot_type=fewshot_type,
+            revision=config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture,
+            num_params=num_params
+        )
+
+def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+    model_result_filepaths = [
+        os.path.join(root, file)
+        for root, _, files in os.walk(results_path)
+        for file in sorted(files, key=lambda x: x.split("_")[-1], reverse=True) if file.endswith(".json")
+    ]
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_name = eval_result.eval_name
+        if eval_name not in eval_results:
+            eval_results[eval_name] = eval_result
+        else:
+            eval_results[eval_name].results.update(eval_result.results)
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict()  # Test if the dict version is complete
+            results.append(v)
+        except KeyError:
+            continue
+
+    return results

src/leaderboard/read_evals_old.py

@@ -0,0 +1,296 @@
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+#from get_model_info import num_params
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, FewShotType
+from src.submission.check_validity import is_model_on_hub
+
+
+@dataclass
+class EvalResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """
+    eval_name: str # org_model_precision (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    revision: str # commit hash, "" if main
+    results: dict
+    average_CPS: str
+    fewshot: int
+    #fewshot_type: str
+    #precision: Precision = Precision.Unknown
+    #model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
+    fewshot_type: FewShotType = FewShotType.Unknown
+    weight_type: WeightType = WeightType.Original # Original or Adapter
+    architecture: str = "Unknown" 
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = "" # submission date of request file
+    still_on_hub: bool = False
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        average_CPS = f"{data.get('average_CPS'):.2f}"
+
+        num_fewshot = config.get("num_fewshot", 0)  # Imposta il valore predefinito a 0
+        try:
+            num_fewshot = int(num_fewshot)  # Converte in intero se possibile
+        except ValueError:
+            num_fewshot = 0  # Se la conversione fallisce, assegna 0
+
+        # Determine the few-shot type (ZS or FS) based on num_fewshot
+        fewshot_type = FewShotType.from_num_fewshot(num_fewshot)  # Use the new
+
+
+        #precision = config.get("precision")
+
+        #print(precision)
+
+        #print(config, num_fewshot)
+
+        # Precision
+        #precision = Precision.from_str(config.get("model_dtype"))
+
+        model_type = config.get("model_type")
+        # Modifica: Convertire model_type in un oggetto Enum (se è un Enum)
+        model_type = ModelType.from_str(model_type) if model_type else None
+
+        #print("=====================", model_type, config.get("model_name"))
+
+        # Initialize num_params with a default value (e.g., 0)
+        num_params = int(0)
+        # Controlla se "num_params_billion" esiste in config e non è null
+        num_params_billion = config.get("num_params_billion")
+        if num_params_billion is not None:
+            num_params = math.ceil(num_params_billion)
+
+        print("^^^^^^^^^^^^^^^^^^^^^^^^^", num_params, config.get("num_params_billion"))
+
+
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        #print(precision.value.name)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            #result_key = f"{model}_{precision.value.name}"
+            result_key = f"{model}_{num_fewshot}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            #result_key = f"{org}_{model}_{precision.value.name}"
+            result_key = f"{org}_{model}_{num_fewshot}"
+        full_model = "/".join(org_and_model)
+
+        still_on_hub, _, model_config = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        )
+        architecture = "?"
+        if model_config is not None:
+            architectures = getattr(model_config, "architectures", None)
+            if architectures:
+                architecture = ";".join(architectures)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+            task = task.value
+
+            '''
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+            '''
+
+            for k, v in data["tasks"].items():
+                #if task.benchmark == k:
+                if task.benchmark[:-2] == k:
+                    # print(k, "==================", v)
+                    # results[task.benchmark] = v[task.cps]
+
+                    #print(task.benchmark, v[task.metric])
+
+                    if "Best Prompt Id" in task.col_name:
+                        results[task.benchmark] = int(v[task.metric_type][-1:])
+                        #print(results[task.benchmark],v[task.metric_type][-1:])
+                    else:
+                        #results[task.benchmark] = round(v[task.metric_type], 2)
+                        # Format the value to 2 decimal places (ensure it's always shown as xx.xx)
+                        results[task.benchmark] = f"{v[task.metric_type]:.2f}"  # Ensure two decimals for display
+
+
+                    #results[task.benchmark + "_" + task.metric] = 1.0
+
+
+                    #results[task.benchmark] = v[task.accuracy]
+                    # print("======", results[task.benchmark])
+                    #results[task.benchmark] = 1.0
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            average_CPS=average_CPS,
+            fewshot_type=fewshot_type,  # Set the fewshot type (ZS or FS)
+            fewshot=num_fewshot,
+            #model_type=model_type,
+            #precision=precision,
+            revision= config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture,
+            num_params=num_params
+        )
+
+    '''
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(f"Could not find request file for {self.org}/{self.model} with precision 
+    '''
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        #average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        average = self.average_CPS
+        fewshot = self.fewshot
+
+        # Ottiene il simbolo di FewShotType in modo simile a ModelType
+        fewshot_type_symbol = (
+            self.fewshot_type.value.symbol if isinstance(self.fewshot_type, FewShotType) else "❓"
+        )
+
+        #("?????", fewshot)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            #AutoEvalColumn.precision.name: self.precision.value.name,
+            #AutoEvalColumn.model_type.name: self.model_type.value.name,
+            #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+
+            #AutoEvalColumn.model_type.name: self.model_type.value.name if self.model_type else "Unknown",
+            #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol if self.model_type else "Unknown",
+
+
+
+            AutoEvalColumn.fewshot_type.name: fewshot_type_symbol,  # Simbolo corretto per fewshot type
+
+
+
+
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.average.name: average,
+            #AutoEvalColumn.fewshot.name: fewshot,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+        }
+
+        for task in Tasks:
+            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+
+        return data_dict
+
+'''
+def get_request_file_for_model(requests_path, model_name, precision):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if (
+                req_content["status"] in ["FINISHED"]
+                and req_content["precision"] == precision.split(".")[-1]
+            ):
+                request_file = tmp_request_file
+    return request_file
+'''
+
+
+
+def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        #eval_result.update_with_request_file(requests_path)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results

src/populate.py

@@ -5,7 +5,7 @@ import pandas as pd
 
 from src.display.formatting import has_no_nan_values, make_clickable_model
 from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
+from src.leaderboard.read_evals_old import get_raw_eval_results
 
 
 def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:

src/submission/check_validity.py

@@ -64,7 +64,7 @@ def get_model_size(model_info: ModelInfo, precision: str):
     try:
         model_size = round(model_info.safetensors["total"] / 1e9, 3)
     except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in example_app.py
 
     size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
     model_size = size_factor * model_size

src/tasks.py

@@ -8,7 +8,6 @@ class Task:
     accuracy: str
     col_name: str
 
-
 NUM_FEWSHOT = 0  # Change with your few shot
 # ---------------------------------------------------
 
@@ -20,12 +19,16 @@ INTRODUCTION_TEXT = """
 Evalita-LLM, a new benchmark designed to evaluate Large Language Models (LLMs) on Italian tasks. The distinguishing and innovative features of Evalita-LLM are the following: (i) all tasks are native Italian, avoiding issues of translating from Italian and potential cultural biases; (ii) in addition to well established multiple-choice tasks, the benchmark includes generative tasks, enabling more natural interaction with LLMs; (iii) all tasks are evaluated against multiple prompts, this way mitigating the model sensitivity to specific prompts and allowing a fairer and objective evaluation.
 """
 
-# Which evaluations are you running? how can people reproduce what you have?
+#MEASURE_DESCRIPTION = "Combined Performance = (1 - (Best_Prompt - Prompt_Average) / 100) * Best_Prompt. Prompt Average = accuracy averaged over the six prompts. Best Prompt = accuracy of the best prompt. Prompt ID = ID of the best prompt (see legend above)"
+MEASURE_DESCRIPTION = "**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = accuracy averaged over the assessed prompts. **Best Prompt** = accuracy of the best prompt. **Prompt ID** = ID of the best prompt (see legend above)."
+
+
+# Tasks Descriptions
 TE_DESCRIPTION = """### Textual Entailment (TE)
     The input are two sentences: the text (T) and the hypothesis (H). The model  has to determine whether the meaning of the hypothesis is logically entailed by the text.
 
 | #   | Prompt | Answer Choices |
-|-----|--------|----------------|
+|-----|------------|--------------|
 | 1   | La frase: '{{text1}}' implica logicamente che la frase: '{{text2}}' sia vera? | ["Sì", "No"] |
 | 2   | Devi risolvere un compito di inferenza semantica. La frase: '{{text1}}' implica logicamente che la frase: '{{text2}}' sia vera? | ["Sì", "No"] |
 | 3   | La frase: '{{text1}}' implica logicamente che la frase: '{{text2}}' sia vera?\\nA: Sì\\nB: No\\nRisposta: | ["A", "B"] |
@@ -33,5 +36,137 @@ TE_DESCRIPTION = """### Textual Entailment (TE)
 | 5   | Frase 1: '{{text1}}' Frase 2: '{{text2}}' | ["La frase 1 implica logicamente che la frase 2 sia vera", "La frase 1 non implica logicamente che la frase 2 sia vera"] |
 | 6   | Devi risolvere un compito di inferenza semantica. Frase 1: '{{text1}}' Frase 2: '{{text2}}' | ["La frase 1 implica logicamente che la frase 2 sia vera", "La frase 1 non implica logicamente che la frase 2 sia vera"] |
 
-Combined Performance = (1 - (Best_Prompt - Prompt_Average) / 100) * Best_Prompt. Prompt Average = accuracy averaged over the six prompts. Best Prompt =  accuracy of the best prompt. Prompt ID =  ID of the best prompt (see legend above)
-"""
+"""
+
+
+SA_DESCRIPTION = """### Sentiment Analysis (SA)
+    The input is a tweet. The model has to determine the sentiment polarity of the text, categorizing it into one of four classes: positive, negative, neutral, or mixed.
+
+| #   | Prompt                                                                                       | Answer Choices               |
+|-----|--------------------------------------------------------------------------------|-----------------------------|
+| 1   | Qual è il sentiment espresso nel seguente tweet: '{{text}}'?                     | ["Positivo", "Negativo", "Neutro", "Misto"] |
+| 2   | Devi svolgere un compito di analisi del sentiment. Qual è il sentiment espresso nel seguente tweet: '{{text}}'? | ["Positivo", "Negativo", "Neutro", "Misto"] |
+| 3   | Qual è il sentiment espresso nel seguente tweet: '{{text}}'?\\nA: Positivo\\nB: Negativo\\nC: Neutro\\nD: Misto\\nRisposta: | ["A", "B", "C", "D"] |
+| 4   | Devi svolgere un compito di analisi del sentiment. Qual è il sentiment espresso nel seguente tweet: '{{text}}'?\\nA: Positivo\\nB: Negativo\\nC: Neutro\\nD: \\nRisposta: | ["A", "B", "C", "D"] |
+| 5   | Il seguente tweet: '{{text}}' esprime un sentiment                                | ["Positivo", "Negativo", "Neutro", "Misto"]  |
+| 6   | Devi svolgere un compito di analisi del sentiment. Il seguente tweet: '{{text}}' esprime un sentiment | ["Positivo", "Negativo", "Neutro", "Misto"] |
+
+"""
+
+
+HS_DESCRIPTION = """### Hate Speech (HS)
+    The input is a tweet. The model has to determine whether the text contains hateful content directed at specific target groups: immigrants, Muslims, or Roma. The output is a binary classification: hateful or not hateful.
+
+| #   | Prompt                                                                                       | Answer Choices                                   |
+|-----|--------------------------------------------------------------------------------|-------------------------------------------------|
+| 1   | C'è incitamento all'odio nel seguente tweet: '{{full_text}}'?                     | ["Falso", "Vero"] |
+| 2   | Devi svolgere un compito di identificazione di incitamento all'odio. C'è incitamento all'odio nel seguente tweet: '{{full_text}}'? | ["Falso", "Vero"] |
+| 3   | C'è incitamento all'odio nel seguente tweet: '{{full_text}}'?\\nA: \\nB: Falso\\nRisposta: | ["B", "A"]                                      |
+| 4   | Devi svolgere un compito di identificazione di incitamento all'odio. C'è incitamento all'odio nel seguente tweet: '{{full_text}}'?\\nA: \\nB: \\nRisposta: | ["B", "A"]                                      |
+| 5   | Il tweet: '{{full_text}}'                                                      | ["non contiene incitamento all'odio", "contiene incitamento all'odio"] |
+| 6   | Devi svolgere un compito di identificazione di incitamento all'odio. Il tweet: '{{full_text}}' | ["non contiene incitamento all'odio", "contiene incitamento all'odio"] |
+
+"""
+
+
+AT_DESCRIPTION = """### Admission Tests (AT)
+    The input is a multiple-choice question with five options (A-E) from Italian medical specialty entrance exams, and the model must identify the correct answer.
+
+| #   | Prompt                                                                                       | Answer Choices               |
+|-----|--------------------------------------------------------------------------------|-----------------------------|
+| 1   | Dato il seguente quesito di medicina: '{{Question}}' qual è la risposta corretta? | ["A", "B", "C", "D", "E"]   |
+| 2   | Devi risolvere un compito di risposte a domande. Dato il seguente quesito di medicina: '{{Question}}' qual è la risposta corretta? | ["A", "B", "C", "D", "E"]   |
+| 3   | Dato il seguente quesito di medicina: '{{Question}}' qual è la risposta corretta?\\nA: {{A}}\\nB: {{B}}\\nC: {{C}}\\nD: {{D}}\\nE: {{E}}\\nRisposta: | ["A", "B", "C", "D", "E"]   |
+| 4   | Devi risolvere un compito a scelta multipla. Dato il seguente caso clinico: '{{background}}', qual è la risposta corretta alla domanda: '{{domanda}}'?\\nA: {{A}}\\nB: {{B}}\\nC: {{C}}\\nD: {{D}}\\nE: {{E}}\\nRisposta:Devi risolvere un compito a scelta multipla. Dato il seguente quesito di medicina: '{{Question}}' qual è la risposta corretta?\\nA: {{A}}\\nB: {{B}}\\nC: {{C}}\\nD: {{D}}\\nE: {{E}}\\nRisposta: | ["A", "B", "C", "D", "E"]   |
+| 5   | Dato il seguente caso clinico: '{{background}}'. La risposta corretta alla domanda: '{{domanda}}' èDato il seguente quesito di medicina '{{Question}}' la risposta corretta è: | ["A", "B", "C", "D", "E"]   |
+| 6   | Devi risolvere un compito di risposte a domande. Dato il seguente quesito di medicina '{{Question}}' la risposta corretta è: | ["A", "B", "C", "D", "E"]   |
+
+"""
+
+WIC_DESCRIPTION = """### Word in Context (WIC)
+    The input consists of a word (w) and two sentences. The model has to determine whether the word w has the same meaning in both sentences. The output is a binary classification: 1 (same meaning) or 0 (different meaning).
+
+| #   | Prompt                                                                                       | Answer Choices                                   |
+|-----|--------------------------------------------------------------------------------|-------------------------------------------------|
+| 1   | La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' ha lo stesso significato della parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}'? | ["No", "Sì"]                         |
+| 2   | Devi determinare se una stessa parola usata in due frasi differenti ha lo stesso significato in entrambi i contesti. La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' ha lo stesso significato della parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}'? | ["No", "Sì"] |
+| 3   | La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' ha lo stesso significato della parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}'?\\nA: Sì\\nB: No\\nRisposta: | ["B", "A"]                                      |
+| 4   | Devi determinare se una stessa parola usata in due frasi differenti ha lo stesso significato in entrambi i contesti. La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' ha lo stesso significato della parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}'?\\nA: \\nB: No\\nRisposta: | ["B", "A"]  |
+| 5   | La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' e la parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}' | ["non hanno lo stesso significato", "hanno lo stesso significato"] |
+| 6   | Devi determinare se una stessa parola usata in due frasi differenti ha lo stesso significato in entrambi i contesti. La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' e la parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}' | ["non hanno lo stesso significato", "hanno lo stesso significato"] |
+
+"""
+
+
+FAQ_DESCRIPTION = """### Frequently Asked Questions & Question Answering (FAQ) 
+    The input is a user query made by customers to the Acquedotto Pugliese service. The model must determine which of the 4 possible answers is the correct response to the question. 
+
+| #   | Prompt                                                                                       | Answer Choices               |
+|-----|--------------------------------------------------------------------------------|-----------------------------|
+| 1   | Rispondi alla seguente domanda: '{{question}}'                                              | {{[A, B, C, D]}}            |
+| 2   | Devi risolvere un compito di risposte a domande. Rispondi alla seguente domanda: '{{question}}' | {{[A, B, C, D]}}            |
+| 3   | Rispondi alla seguente domanda: '{{question}}'\\nA: {{A}}\\nB: {{B}}\\nC: {{C}}\\nD: {{D}}\\nRisposta: | ["A", "B", "C", "D"] |
+| 4   | Devi risolvere un compito a scelta multipla. Rispondi alla seguente domanda: '{{question}}'\\nA: {{A}}\\nB: {{B}}\\nC: {{C}}\\nD: {{D}}\\nRisposta: | ["A", "B", "C", "D"] |
+| 5   | La risposta alla domanda: '{{question}}' è:                                                | {{[A, B, C, D]}}            |
+| 6   | Devi risolvere un compito di risposte a domande. La risposta alla domanda: '{{question}}' è: | {{[A, B, C, D]}}  |
+
+"""
+
+
+LS_DESCRIPTION = """### Lexical Substitution (LS)
+    The input is a sentence containing a target word (w). The model has to replace the target word w with its most suitable synonyms that are contextually relevant. 
+
+| #   | Prompt                                                                                       |
+|-----|--------------------------------------------------------------------------------|
+| 1   | Trova 10 parole che possono sostituire la parola racchiusa tra i marcatori `<head>` nella seguente frase: '{{context}}', mantenendo lo stesso significato. Elenca i lemmi (forme base) di queste parole, separandoli con una virgola, ad esempio: lemma1, lemma2, lemma3, lemma4, lemma5. Non aggiungere commenti o altro testo. Risposta: |
+| 2   | Devi risolvere un compito di sostituzione lessicale. Trova 10 parole che possono sostituire la parola racchiusa tra i marcatori `<head>` nella seguente frase: '{{context}}', mantenendo lo stesso significato. Elenca i lemmi (forme base) di queste parole, separandoli con una virgola, ad esempio: lemma1, lemma2, lemma3, lemma4, lemma5. Non aggiungere commenti o altro testo. Risposta: |
+
+"""
+
+
+SU_DESCRIPTION = """### Summarization (SUM)
+    The input is a news article. The model has to generate a concise summary of the input text, capturing the key information and main points.
+
+| #   | Prompt                                                                                       |
+|-----|--------------------------------------------------------------------------------|
+| 1   | Riassumi il seguente articolo di giornale: '{{source}}'\\nRiassunto:             |
+| 2   | Devi risolvere un compito di sintesi automatica del testo. Riassumi il seguente articolo di giornale: '{{source}}'\\nRiassunto: |
+
+"""
+
+
+NER_DESCRIPTION = """### Named Entity Recognition (NER)
+    The input is a sentence. The model has to identify and classify Named Entities into predefined categories such as person, organization, and location.
+
+| #   | Prompt                                                                                       |
+|-----|--------------------------------------------------------------------------------|
+| 1   | Estrai tutte le entità di tipo PER (persona), LOC (luogo) e ORG (organizzazione) dal testo seguente. Riporta ogni entità con il formato: Entità$Tipo, separando ciascuna coppia con ','. Se non ci sono entità da estrarre, rispondi con '&&NOENT&&'.\\nTesto: '{{text}}'\\nEntità: |
+| 2   | Devi svolgere un compito di riconoscimento delle entità nei testi. Estrai tutte le entità di tipo PER (persona), LOC (luogo) e ORG (organizzazione) dal testo seguente. Riporta ogni entità con il formato: Entità$Tipo, separando ciascuna coppia con ','. Se non ci sono entità da estrarre, rispondi con '&&NOENT&&'.\\nTesto: '{{text}}'\\nEntità: |
+
+"""
+
+
+REL_DESCRIPTION = """### Relation Extraction (REL) 
+    The task involves analyzing clinical text to extract relationships between laboratory test results (e.g., blood pressure) and the tests or procedures that produced them (e.g., blood pressure test).
+
+| #   | Prompt                                                                                       |
+|-----|--------------------------------------------------------------------------------|
+| 1   | Dato un documento medico devi estrarre tutte le misurazioni degli esami medici presenti. Riporta ogni relazione nel formato: misurazione$esame, separando ciascuna coppia con '%'. Se non ci sono relazioni da estrarre, rispondi con '&&NOREL&&'.\\nTesto: '{{text}}'\\nRelazioni: |
+| 2   | Devi svolgere un compito di estrazione di relazioni da documenti medici. Dato un documento medico devi estrarre tutte le misurazioni degli esami medici presenti. Riporta ogni relazione nel formato: misurazione$esame, separando ciascuna coppia con '%'. Se non ci sono relazioni da estrarre, rispondi con '&&NOREL&&'.\\nTesto: '{{text}}'\\nRelazioni: |
+
+"""
+
+
+# Create a dictionary to map task names to their descriptions
+TASK_DESCRIPTIONS = {
+    "TE": TE_DESCRIPTION,
+    "SA": SA_DESCRIPTION,
+    "HS": HS_DESCRIPTION,
+    "AT": AT_DESCRIPTION,
+    "WIC": WIC_DESCRIPTION,
+    "FAQ": FAQ_DESCRIPTION,
+    "LS": LS_DESCRIPTION,
+    "SU": SU_DESCRIPTION,
+    "NER": NER_DESCRIPTION,
+    "REL": REL_DESCRIPTION
+}