Add Size field to the leaderboard

app.py

@@ -117,7 +117,7 @@ def barplot_mean_few_minus_zero_shot(dataframe, tasks=None):
 
 def boxplot_per_task(dataframe=None, baselines=None):
 
-    print(dataframe.columns)
+    #print(dataframe.columns)
 
     tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
 
@@ -399,7 +399,7 @@ def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
     sorted_dataframe = dataframe.sort_values(by="Avg. Comb. Perf. ⬆️", ascending=False)
 
     sorted_dataframe = sorted_dataframe.reset_index(drop=True)
-    sorted_dataframe["rank"] = sorted_dataframe.index + 1
+    sorted_dataframe["Rank"] = sorted_dataframe.index + 1
 
     # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo
     large_medal_fs_assigned = False
@@ -415,26 +415,26 @@ def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
 
     for _, row in sorted_dataframe.iterrows():
         if row['IS_FS']:  # 5-Few-Shot
-            if row["#Params (B)"] > 50 and not large_medal_fs_assigned:
-                new_model_column.append(f"{row['Model']} 1️⃣0️⃣0️⃣🅱️🏆")
+            if row["Size"] == "🔵🔵🔵" and not large_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🏆")
                 large_medal_fs_assigned = True
-            elif 10 < row["#Params (B)"] <= 50 and not medium_medal_fs_assigned:
-                new_model_column.append(f"{row['Model']} 5️⃣0️⃣🅱️🏆")
+            elif row["Size"] == "🔵🔵" and not medium_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🏆")
                 medium_medal_fs_assigned = True
-            elif row["#Params (B)"] <= 10 and not small_medal_fs_assigned:
-                new_model_column.append(f"{row['Model']} 1️⃣0️⃣🅱️🏆")
+            elif row["Size"] == "🔵" and not small_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🏆")
                 small_medal_fs_assigned = True
             else:
                 new_model_column.append(row["Model"])
         else:  # 0-Shot
-            if row["#Params (B)"] > 50 and not large_medal_0shot_assigned:
-                new_model_column.append(f"{row['Model']} 1️⃣0️⃣0️⃣🅱️🎖️")
+            if row["Size"] == "🔵🔵🔵" and not large_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🎖️")
                 large_medal_0shot_assigned = True
-            elif 10 < row["#Params (B)"] <= 50 and not medium_medal_0shot_assigned:
-                new_model_column.append(f"{row['Model']} 5️⃣0️⃣🅱️🎖️")
+            elif row["Size"] == "🔵🔵" and not medium_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🎖️")
                 medium_medal_0shot_assigned = True
-            elif row["#Params (B)"] <= 10 and not small_medal_0shot_assigned:
-                new_model_column.append(f"{row['Model']} 1️⃣0️⃣🅱️🎖️")
+            elif row["Size"] == "🔵" and not small_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🎖️")
                 small_medal_0shot_assigned = True
             else:
                 new_model_column.append(row["Model"])
@@ -488,7 +488,7 @@ def update_task_leaderboard(dataframe, default_selection=None, hidden_columns=No
 
     # aggiungo la colonna rank in base alla posizione
     sorted_dataframe = sorted_dataframe.reset_index(drop=True)
-    sorted_dataframe["rank"] = sorted_dataframe.index + 1
+    sorted_dataframe["Rank"] = sorted_dataframe.index + 1
 
     # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo
     large_medal_fs_assigned = False
@@ -504,26 +504,26 @@ def update_task_leaderboard(dataframe, default_selection=None, hidden_columns=No
 
     for _, row in sorted_dataframe.iterrows():
         if row['IS_FS']:  # 5-Few-Shot
-            if row["#Params (B)"] > 30 and not large_medal_fs_assigned:
-                new_model_column.append(f"{row['Model']} 1️⃣0️⃣0️⃣🅱️🏆")
+            if row["Size"] == "🔵🔵🔵" and not large_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🏆")
                 large_medal_fs_assigned = True
-            elif 10 < row["#Params (B)"] <= 30 and not medium_medal_fs_assigned:
-                new_model_column.append(f"{row['Model']} 5️⃣0️⃣🅱️🏆")
+            elif row["Size"] == "🔵🔵" and not medium_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🏆")
                 medium_medal_fs_assigned = True
-            elif row["#Params (B)"] <= 10 and not small_medal_fs_assigned:
-                new_model_column.append(f"{row['Model']} 1️⃣0️⃣🅱️🏆")
+            elif row["Size"] == "🔵" and not small_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🏆")
                 small_medal_fs_assigned = True
             else:
                 new_model_column.append(row["Model"])
         else:  # 0-Shot
-            if row["#Params (B)"] > 30 and not large_medal_0shot_assigned:
-                new_model_column.append(f"{row['Model']} 1️⃣0️⃣0️⃣🅱️🎖️")
+            if row["Size"] == "🔵🔵🔵" and not large_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🎖️")
                 large_medal_0shot_assigned = True
-            elif 10 < row["#Params (B)"] <= 30 and not medium_medal_0shot_assigned:
-                new_model_column.append(f"{row['Model']} 5️⃣0️⃣🅱️🎖️")
+            elif row["Size"] == "🔵🔵" and not medium_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🎖️")
                 medium_medal_0shot_assigned = True
-            elif row["#Params (B)"] <= 10 and not small_medal_0shot_assigned:
-                new_model_column.append(f"{row['Model']} 1️⃣0️⃣🅱️🎖️")
+            elif row["Size"] == "🔵" and not small_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🎖️")
                 small_medal_0shot_assigned = True
             else:
                 new_model_column.append(row["Model"])
@@ -646,8 +646,8 @@ with demo:
 
             leaderboard = init_leaderboard(
                 LEADERBOARD_DF,
-                default_selection=['rank', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"],
-                hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['rank', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]]
+                default_selection=['Rank', 'Size', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]]
             )
 
             gr.HTML(
@@ -693,8 +693,8 @@ with demo:
 
                 leaderboard = update_task_leaderboard(
                     LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average", f"{task} Prompt Std": "Prompt Std", f"{task} Best Prompt": "Best Prompt", f"{task} Best Prompt Id": "Best Prompt Id", task: "Combined Performance"}),
-                    default_selection=['rank', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'],
-                    hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['rank', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id']]
+                    default_selection=['Rank', 'Size', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'],
+                    hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id']]
                 )
 
         # About tab
@@ -713,10 +713,10 @@ with demo:
                                                    f"{task} Best Prompt": "Best Prompt",
                                                    f"{task} Best Prompt Id": "Best Prompt Id",
                                                    task: "Combined Performance"}),
-                    default_selection=['rank', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt',
+                    default_selection=['Rank', 'Size', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt',
                                        'Best Prompt Id'],
                     hidden_columns=[col for col in LEADERBOARD_DF.columns if
-                                    col not in ['rank', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std',
+                                    col not in ['Rank', 'Size', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std',
                                                 'Best Prompt', 'Best Prompt Id']]
                 )
 

src/display/utils.py

@@ -25,7 +25,8 @@ auto_eval_column_dict = []
 # Init
 #auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 
-auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent("rank", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent("Rank", "number", True, never_hidden=True)])
+auto_eval_column_dict.append(["size_symbol", ColumnContent, ColumnContent("Size", "number", True, never_hidden=True)])
 
 auto_eval_column_dict.append(["fewshot_symbol", ColumnContent, ColumnContent("FS", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["is_5fewshot", ColumnContent, ColumnContent("IS_FS", "bool", True)])
@@ -99,7 +100,7 @@ class FewShotDetails:
     symbol: str = ""  # emoji
 
 class FewShotType(Enum):
-    ZS = FewShotDetails(name="zero-shot", symbol="0️⃣")
+    ZS = FewShotDetails(name="zero-shot", symbol="🅾️")
     FS = FewShotDetails(name="5-few-shot", symbol="5️⃣")
     Unknown = FewShotDetails(name="unknown", symbol="❓")
 
@@ -115,6 +116,30 @@ class FewShotType(Enum):
             return FewShotType.FS
         return FewShotType.Unknown
 
+@dataclass
+class SizeDetails:
+    name: str
+    symbol: str = ""  # emoji
+
+class SizeType(Enum):
+    SMALL = SizeDetails(name="small", symbol="🔵")
+    MEDIUM = SizeDetails(name="medium", symbol="🔵🔵")
+    LARGE = SizeDetails(name="large", symbol="🔵🔵🔵")
+    Unknown = SizeDetails(name="unknown", symbol="❓")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def num2type(size):
+        """Determines FewShotType based on num_fewshot."""
+        if size <= 10:
+            return SizeType.SMALL
+        elif size > 10 and size <= 50:
+            return SizeType.MEDIUM
+        else:
+            return SizeType.LARGE
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")

src/leaderboard/read_evals.py

@@ -11,7 +11,7 @@ from datetime import datetime
 
 #from get_model_info import num_params
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, FewShotType
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, FewShotType, SizeType
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -36,7 +36,8 @@ class EvalResult:
     num_params: int = 0
     date: str = "" # submission date of request file
     still_on_hub: bool = False
-    rank: int = field(default=0)  # 👈 nuovo campo con default = 0
+    rank: int = 0#str = field(default=0)  # nuovo campo con default = 0
+    size_symbol: SizeType = SizeType.Unknown
 
     @classmethod
     def init_from_json_file(self, json_filepath):
@@ -52,6 +53,8 @@ class EvalResult:
         # Get number of fewshot
         fewshot = config.get("num_fewshot", False)
 
+        rank = 0
+
         try:
             if fewshot == "5":
                 is_5fewshot = True
@@ -68,6 +71,8 @@ class EvalResult:
         if num_params_billion is not None:
             num_params = math.ceil(num_params_billion)
 
+        size_symbol = SizeType.num2type(num_params)
+
         # Get model and org
         org_and_model = config.get("model_name", config.get("model_args", None))
         org_and_model = org_and_model.split("/", 1)
@@ -121,7 +126,8 @@ class EvalResult:
             still_on_hub=still_on_hub,
             architecture=architecture,
             num_params=num_params,
-            rank = 0,
+            rank = rank,
+            size_symbol=size_symbol
             #submitted_time=config.get("submitted_time", ""),
         )
 
@@ -151,6 +157,10 @@ class EvalResult:
             self.fewshot_symbol.value.symbol if isinstance(self.fewshot_symbol, FewShotType) else "❓"
         )
 
+        size_symbol = (
+            self.size_symbol.value.symbol if isinstance(self.size_symbol, SizeType) else "❓"
+        )
+
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             #AutoEvalColumn.precision.name: self.precision.value.name,
@@ -169,7 +179,8 @@ class EvalResult:
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-            "rank": self.rank
+            AutoEvalColumn.rank.name: self.rank,
+            AutoEvalColumn.size_symbol.name: size_symbol
         }
 
         for task in Tasks: