Small changes

app.py

@@ -10,6 +10,7 @@ from src.display.utils import BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoE
 from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
+import random
 
 # Define task metadata (icons, names, descriptions)
 TASK_METADATA_MULTIPLECHOICE = {
@@ -35,7 +36,10 @@ def restart_space():
 
 
 def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
-    """Initialize and return a leaderboard."""
+    """
+    Initialize and return the leaderboard when it is first loaded or when 'benchmark' is selected.
+    The table is sorted based on the "Avg. Combined Performance" field.
+    """
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
 
@@ -52,13 +56,50 @@ def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
         search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
         hide_columns=hidden_columns or [c.name for c in field_list if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.fewshot_type.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)")
+            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)"),
+            #ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)",
+            #             default=[["0️⃣", "0️⃣"]]),
         #   ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=150, label="Select the number of parameters (B)"),
         ],
-        bool_checkboxgroup_label="Hide models",
+        #filter_columns=[
+        #    ColumnFilter("IS_FS", type="checkbox", default=False, label="5-Few-Shot")
+        #    #ColumnFilter("FS", type="dropdown", label="5-Few-Shot")
+        #],
+        bool_checkboxgroup_label="Evaluation Mode",
         interactive=False,
     )
 
+def update_task_leaderboard(dataframe, default_selection=None, hidden_columns=None):
+    """
+    Update and return the leaderboard when a specific task is selected.
+    The table is sorted based on the "Combined Performance" field.
+    """
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    sorted_dataframe = dataframe.sort_values(by="Combined Performance", ascending=False)
+
+    #print(sorted_dataframe['Combined Performance'])
+
+    field_list = fields(AutoEvalColumn)
+
+    return Leaderboard(
+        value=sorted_dataframe,
+        datatype=[c.type for c in field_list],
+        #select_columns=SelectColumns(
+        #    default_selection=default_selection or [c.name for c in field_list if c.displayed_by_default],
+        #    cant_deselect=[c.name for c in field_list if c.never_hidden],
+        #    label="Select Columns to Display:",
+        #),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=hidden_columns or [c.name for c in field_list if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)"),
+        ],
+        bool_checkboxgroup_label="Evaluation Mode",
+        interactive=False
+    )
+
 '''
 # Helper function for leaderboard initialization
 def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
@@ -137,7 +178,7 @@ with demo:
                 task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
                 gr.Markdown(task_description, elem_classes="markdown-text")
 
-                leaderboard = init_leaderboard(
+                leaderboard = update_task_leaderboard(
                     LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average", f"{task} Best Prompt": "Best Prompt", f"{task} Best Prompt Id": "Best Prompt Id", task: "Combined Performance"}),
                     default_selection=['FS', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id'],
                     hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['FS', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id']]
@@ -153,7 +194,7 @@ with demo:
                 task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
                 gr.Markdown(task_description, elem_classes="markdown-text")
 
-                leaderboard = init_leaderboard(
+                leaderboard = update_task_leaderboard(
                     LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average",
                                                    f"{task} Best Prompt": "Best Prompt",
                                                    f"{task} Best Prompt Id": "Best Prompt Id",
@@ -175,4 +216,5 @@ scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
 
 # Launch the app with concurrent queueing
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch(debug=True,  # Enable Gradio debug mode
+        show_error=True)

src/display/utils.py

@@ -25,7 +25,8 @@ auto_eval_column_dict = []
 # Init
 #auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 
-auto_eval_column_dict.append(["fewshot_type", ColumnContent, ColumnContent("FS", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["fewshot_symbol", ColumnContent, ColumnContent("FS", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["is_5fewshot", ColumnContent, ColumnContent("IS_FS", "bool", True)])
 
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #auto_eval_column_dict.append(["fewshot", ColumnContent, ColumnContent("Few-Shot", "str", True)])
@@ -103,11 +104,11 @@ class FewShotType(Enum):
         return f"{self.value.symbol}{separator}{self.value.name}"
 
     @staticmethod
-    def from_num_fewshot(num_fewshot):
+    def from_num_fewshot(is_5fewshot):
         """Determines FewShotType based on num_fewshot."""
-        if num_fewshot == 0:
+        if is_5fewshot is False:
             return FewShotType.ZS
-        if num_fewshot == 5:
+        elif is_5fewshot is True:
             return FewShotType.FS
         return FewShotType.Unknown
 

src/leaderboard/read_evals.py

@@ -25,8 +25,8 @@ class EvalResult:
     revision: str # commit hash, "" if main
     results: Dict[str, Union[float, int]]  # float o int
     average_CPS: float
-    fewshot: int
-    fewshot_type: FewShotType = FewShotType.Unknown
+    is_5fewshot: bool
+    fewshot_symbol: FewShotType = FewShotType.Unknown
     weight_type: WeightType = WeightType.Original # Original or Adapter
     architecture: str = "Unknown" 
     license: str = "?"
@@ -47,13 +47,17 @@ class EvalResult:
         # Ottieni average_CPS come float
         average_CPS = float(data.get('average_CPS', 0.0))  # 0.0 come valore di default
 
-        num_fewshot = config.get("num_fewshot", 0)  # Imposta il valore predefinito a 0
+        fewshot = config.get("num_fewshot", False)  # Imposta il valore predefinito a 0
+
         try:
-            num_fewshot = int(num_fewshot)  # Converte in intero se possibile
+            if fewshot == "5":
+                is_5fewshot = True
+            else:
+                is_5fewshot = False# Converte in intero se possibile
         except ValueError:
-            num_fewshot = 0  # Se la conversione fallisce, assegna 0
+            is_5fewshot = False  # Se la conversione fallisce, assegna 0
         # Determine the few-shot type (ZS or FS) based on num_fewshot
-        fewshot_type = FewShotType.from_num_fewshot(num_fewshot)  # Use the new
+        fewshot_symbol = FewShotType.from_num_fewshot(is_5fewshot)  # Use the new
 
         num_params = int(0)
         num_params_billion = config.get("num_params_billion")
@@ -68,12 +72,12 @@ class EvalResult:
             org = None
             model = org_and_model[0]
             #result_key = f"{model}_{precision.value.name}"
-            result_key = f"{model}_{num_fewshot}"
+            result_key = f"{model}_{is_5fewshot}"
         else:
             org = org_and_model[0]
             model = org_and_model[1]
             #result_key = f"{org}_{model}_{precision.value.name}"
-            result_key = f"{org}_{model}_{num_fewshot}"
+            result_key = f"{org}_{model}_{is_5fewshot}"
         full_model = "/".join(org_and_model)
 
         still_on_hub, _, model_config = is_model_on_hub(
@@ -107,8 +111,8 @@ class EvalResult:
             model=model,
             results=results,
             average_CPS=average_CPS,
-            fewshot_type=fewshot_type,
-            fewshot=num_fewshot,
+            fewshot_symbol=fewshot_symbol,
+            is_5fewshot=is_5fewshot,
             revision= config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture,
@@ -137,8 +141,8 @@ class EvalResult:
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         average = self.average_CPS
 
-        fewshot_type_symbol = (
-            self.fewshot_type.value.symbol if isinstance(self.fewshot_type, FewShotType) else "❓"
+        fewshot_symbol = (
+            self.fewshot_symbol.value.symbol if isinstance(self.fewshot_symbol, FewShotType) else "❓"
         )
 
         data_dict = {
@@ -148,13 +152,13 @@ class EvalResult:
             #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             #AutoEvalColumn.model_type.name: self.model_type.value.name if self.model_type else "Unknown",
             #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol if self.model_type else "Unknown",
-            AutoEvalColumn.fewshot_type.name: fewshot_type_symbol,  # Simbolo corretto per fewshot type
+            AutoEvalColumn.fewshot_symbol.name: fewshot_symbol,
             AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average.name: average,
-            #AutoEvalColumn.fewshot.name: fewshot,
+            AutoEvalColumn.is_5fewshot.name: self.is_5fewshot,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,