wip added primary category tasks

src/about.py

@@ -10,10 +10,20 @@ class Task:
 
 # Select your tasks here
 # ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+if False:
+    class Tasks(Enum):
+        # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+        task0 = Task("anli_r1", "acc", "ANLI")
+        task1 = Task("logiqa", "acc_norm", "LogiQA")
+else:
+    class Tasks(Enum):
+        # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+        # NOTE: the task_key must be unique
+        task0 = Task("default_primary_subfield_accuracy", "primary_subfield_accuracy", "Accuracy of predicting the released primary category on the default split")
+        task1 = Task("default_primary_top_3_tpr", "primary_top_3_tpr", "Occurrence of the released primary category in the top-3 predictions on the default split")
+        task2 = Task("default_primary_top_5_tpr", "primary_top_5_tpr", "Occurrence of the released primary category in the top-5 predictions on the default split")
+        task3 = Task("default_primary_top_10_tpr", "primary_top_10_tpr", "Occurrence of the released primary category in the top-10 predictions on the default split")
+        # task1 = Task("all2023_v2", "acc", "Acc on all 2023 papers")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass, make_dataclass
+from dataclasses import dataclass, make_dataclass, field
 from enum import Enum
 
 import pandas as pd
@@ -21,27 +21,46 @@ class ColumnContent:
     never_hidden: bool = False
 
 ## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+if False:
+    auto_eval_column_dict = []
+    # Init
+    auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+    auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+    #Scores
+    auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+    for task in Tasks:
+        auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    # Model information
+    auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+    auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+    auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+    auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+    auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+    auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+    auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+    auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+    auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+    # We use make dataclass to dynamically fill the scores from Tasks
+    AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+@dataclass(frozen=True)
+class AutoEvalColumn:
+    model_type_symbol = ColumnContent("T", "str", True, never_hidden=True)
+    model = ColumnContent("Model", "markdown", True, never_hidden=True)
+    average = ColumnContent("Average ⬆️", "number", True)
+    model_type = ColumnContent("Type", "str", False)
+    architecture = ColumnContent("Architecture", "str", False)
+    weight_type = ColumnContent("Weight type", "str", False, True)
+    precision = ColumnContent("Precision", "str", False)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    still_on_hub = ColumnContent("Available on the hub", "bool", False)
+    revision = ColumnContent("Model sha", "str", False, False)
+    # Dynamically add task columns
+    def __init__(self):
+        for task in Tasks:
+            setattr(self.__class__, task.name, ColumnContent(task.value.col_name, "number", True))
 
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
@@ -107,4 +126,5 @@ EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-
+# NOTE: add the benchmark columns to the COLS since they are dynamically added
+COLS += BENCHMARK_COLS

src/envs.py

@@ -11,8 +11,8 @@ OWNER = "mlcore"
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/arxiv-classifier-leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+QUEUE_REPO = f"{OWNER}/arxiv-classifier-leaderboard-requests"
+RESULTS_REPO = f"{OWNER}/arxiv-classifier-leaderboard-results"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")

src/leaderboard/read_evals.py

@@ -176,6 +176,9 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        # TODO: populate requests repo with request files corresponding to llama3-8b_primary
+        # Current output of `python app.py`:
+        # Could not find request file for None/llama3-8b_primary with precision ?
         eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
@@ -187,6 +190,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
 
     results = []
     for v in eval_results.values():
+        # import pdb; pdb.set_trace()
         try:
             v.to_dict() # we test if the dict version is complete
             results.append(v)