sort results by date

src/about.py

@@ -7,35 +7,36 @@ class Task:
     metric: str
     col_name: str
     type: str
+    baseline: float = 0.0
 
 
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    # task2 = Task("belebele_pol_Latn", "acc,none", "belebele_pol_Latn", "multiple_choice")
-    task3 = Task("polemo2_in", "exact_match,score-first", "polemo2-in_g", "generate_until")
-    task4 = Task("polemo2_in_multiple_choice", "acc,none", "polemo2-in_mc", "multiple_choice")
-    task5 = Task("polemo2_out", "exact_match,score-first", "polemo2-out_g", "generate_until")
-    task6 = Task("polemo2_out_multiple_choice", "acc,none", "polemo2-out_mc", "multiple_choice")
-    task7 = Task("polish_8tags_multiple_choice", "acc,none", "8tags_mc", "multiple_choice")
-    task8 = Task("polish_8tags_regex", "exact_match,score-first", "8tags_g", "generate_until")
-    task9a = Task("polish_belebele_mc", "acc,none", "belebele_mc", "multiple_choice")
-    task9 = Task("polish_belebele_regex", "exact_match,score-first", "belebele_g", "generate_until")
-    task10 = Task("polish_dyk_multiple_choice", "f1,none", "dyk_mc", "multiple_choice")
-    task11 = Task("polish_dyk_regex", "f1,score-first", "dyk_g", "generate_until")
-    task12 = Task("polish_ppc_multiple_choice", "acc,none", "ppc_mc", "multiple_choice")
-    task13 = Task("polish_ppc_regex", "exact_match,score-first", "ppc_g", "generate_until")
-    task14 = Task("polish_psc_multiple_choice", "f1,none", "psc_mc", "multiple_choice")
-    task15 = Task("polish_psc_regex", "f1,score-first", "psc_g", "generate_until")
-    task16 = Task("polish_cbd_multiple_choice", "f1,none", "cbd_mc", "multiple_choice")
-    task17 = Task("polish_cbd_regex", "f1,score-first", "cbd_g", "generate_until")
-    task18 = Task("polish_klej_ner_multiple_choice", "acc,none", "klej_ner_mc", "multiple_choice")
-    task19 = Task("polish_klej_ner_regex", "exact_match,score-first", "klej_ner_g", "generate_until")
+    # task2 = Task("belebele_pol_Latn", "acc,none", "belebele_pol_Latn", "multiple_choice", 0.279)
+    task3 = Task("polemo2_in", "exact_match,score-first", "polemo2-in_g", "generate_until", 0.416)
+    task4 = Task("polemo2_in_multiple_choice", "acc,none", "polemo2-in_mc", "multiple_choice", 0.416)
+    task5 = Task("polemo2_out", "exact_match,score-first", "polemo2-out_g", "generate_until", 0.368)
+    task6 = Task("polemo2_out_multiple_choice", "acc,none", "polemo2-out_mc", "multiple_choice", 0.368)
+    task7 = Task("polish_8tags_multiple_choice", "acc,none", "8tags_mc", "multiple_choice", 0.143)
+    task8 = Task("polish_8tags_regex", "exact_match,score-first", "8tags_g", "generate_until", 0.143)
+    task9a = Task("polish_belebele_mc", "acc,none", "belebele_mc", "multiple_choice", 0.279)
+    task9 = Task("polish_belebele_regex", "exact_match,score-first", "belebele_g", "generate_until", 0.279)
+    task10 = Task("polish_dyk_multiple_choice", "f1,none", "dyk_mc", "multiple_choice", 0.289)
+    task11 = Task("polish_dyk_regex", "f1,score-first", "dyk_g", "generate_until", 0.289)
+    task12 = Task("polish_ppc_multiple_choice", "acc,none", "ppc_mc", "multiple_choice", 0.419)
+    task13 = Task("polish_ppc_regex", "exact_match,score-first", "ppc_g", "generate_until", 0.419)
+    task14 = Task("polish_psc_multiple_choice", "f1,none", "psc_mc", "multiple_choice", 0.466)
+    task15 = Task("polish_psc_regex", "f1,score-first", "psc_g", "generate_until", 0.466)
+    task16 = Task("polish_cbd_multiple_choice", "f1,none", "cbd_mc", "multiple_choice", 0.149)
+    task17 = Task("polish_cbd_regex", "f1,score-first", "cbd_g", "generate_until", 0.149)
+    task18 = Task("polish_klej_ner_multiple_choice", "acc,none", "klej_ner_mc", "multiple_choice", 0.343)
+    task19 = Task("polish_klej_ner_regex", "exact_match,score-first", "klej_ner_g", "generate_until", 0.343)
     task20 = Task("polish_poleval2018_task3_test_10k", "word_perplexity,none", "poleval2018_task3_test_10k", "other")
-    task21 = Task("polish_polqa_reranking_multiple_choice", "acc,none", "polqa_reranking_mc", "other") # multiple_choice
-    task22 = Task("polish_polqa_open_book", "levenshtein,none", "polqa_open_book_g", "other") # generate_until
-    task23 = Task("polish_polqa_closed_book", "levenshtein,none", "polqa_closed_book_g", "other") # generate_until
+    # task21 = Task("polish_polqa_reranking_multiple_choice", "acc,none", "polqa_reranking_mc", "multiple_choice", 0.5335588952710677) # multiple_choice
+    # task22 = Task("polish_polqa_open_book", "levenshtein,none", "polqa_open_book_g", "generate_until", 0.0) # generate_until
+    # task23 = Task("polish_polqa_closed_book", "levenshtein,none", "polqa_closed_book_g", "generate_until", 0.0) # generate_until
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/leaderboard/read_evals.py

@@ -14,26 +14,28 @@ from src.submission.check_validity import is_model_on_hub
 
 NUM_FEWSHOT = 0
 
+
 @dataclass
 class EvalResult:
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
+    org: str
     model: str
-    revision: str # commit hash, "" if main
+    revision: str  # commit hash, "" if main
     results: dict
     precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
+    model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original  # Original or Adapter
+    architecture: str = "Unknown"
     license: str = "?"
     lang: str = "?"
     likes: int = 0
     num_params: int = 0
-    date: str = "" # submission date of request file
+    date: str = ""  # submission date of request file
     still_on_hub: bool = False
     n_shot: NShotType = NShotType.n0
     org_and_model: str = ""
+    start_date: float = 0
 
     @classmethod
     def init_from_json_file(self, json_filepath, n_shot_num):
@@ -43,6 +45,7 @@ class EvalResult:
 
         config = data.get("config")
         n_shot = data.get("n-shot")
+        start_date = data.get("date", 0)
 
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
@@ -54,14 +57,17 @@ class EvalResult:
 
         if re.match(r"^pretrained=(.*/(plgkwrobel|plggspkl)/)(models/)?", org_and_model):
             org_and_model = re.sub(r"^pretrained=(.*/(plgkwrobel|plggspkl)/)(models/)?", SPICHLERZ_ORG, org_and_model)
-            org_and_model = org_and_model.replace(",dtype=bfloat16", "")
 
-        org_and_model=org_and_model.replace("models/hf_v7_e1", "APT3-1B-Instruct-e1")
-        org_and_model=org_and_model.replace("models/hf_v7_e2", "APT3-1B-Instruct-e2")
+        org_and_model = org_and_model.replace(",dtype=bfloat16", "")
+        org_and_model = org_and_model.replace(",dtype=float16", "")
+
+        org_and_model = org_and_model.replace("models/hf_v7_e1", "APT3-1B-Instruct-e1")
+        org_and_model = org_and_model.replace("models/hf_v7_e2", "APT3-1B-Instruct-e2")
 
         org_and_model = re.sub(r"^pretrained=", "", org_and_model)
         org_and_model = org_and_model.replace(",trust_remote_code=True", "")
         org_and_model = re.sub(",prefix_token_id=\d+", "", org_and_model)
+        org_and_model = re.sub("/$", "", org_and_model)
 
         org_and_model = org_and_model.split("/", 1)
 
@@ -90,7 +96,8 @@ class EvalResult:
             task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k and n_shot.get(k, -1) == n_shot_num])
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if
+                             task.benchmark == k and n_shot.get(k, -1) == n_shot_num])
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
@@ -98,7 +105,8 @@ class EvalResult:
                 mean_acc = np.mean(accs)
             else:
                 mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
+            results[task.benchmark] = (mean_acc, start_date)
+            # results[task.benchmark] = mean_acc
 
         return self(
             eval_name=result_key,
@@ -106,27 +114,27 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
+            precision=precision,
+            revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             architecture=architecture,
             n_shot=NShotType.from_str(n_shot_num),
-            org_and_model=orig_org_and_model
+            org_and_model=orig_org_and_model,
+            start_date=start_date
         )
 
     def update_with_metadata(self, metadata):
-        #print('UPDATE', self.full_model, self.model, self.eval_name)
+        # print('UPDATE', self.full_model, self.model, self.eval_name)
         try:
-            meta=metadata[self.full_model]
+            meta = metadata[self.full_model]
             self.model_type = ModelType.from_str(meta.get("type", "?"))
             self.num_params = meta.get("params", 0)
             self.license = meta.get("license", "?")
             self.lang = meta.get("lang", "?")
-            #TODO desc name
+            # TODO desc name
         except KeyError:
             print(f"Could not find metadata for {self.full_model}")
 
-
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
         return
@@ -149,12 +157,18 @@ class EvalResult:
         g_tasks = [task.value.benchmark for task in Tasks if task.value.type == "generate_until"]
         mc_tasks = [task.value.benchmark for task in Tasks if task.value.type == "multiple_choice"]
         all_tasks = g_tasks + mc_tasks
-        average = sum([v for task,v in self.results.items() if v is not None and task in all_tasks]) / len(all_tasks)
-        average_g = sum([v for task,v in self.results.items() if v is not None and task in g_tasks]) / len(g_tasks)
-        average_mc = sum([v for task,v in self.results.items() if v is not None and task in mc_tasks]) / len(mc_tasks)
 
+        baselines = {task.value.benchmark: task.value.baseline*100 for task in Tasks}
 
-        data_dict={}
+        average = sum([v for task, v in self.results.items() if v is not None and task in all_tasks]) / len(all_tasks)
+        average_g = sum([v for task, v in self.results.items() if v is not None and task in g_tasks]) / len(g_tasks)
+        average_mc = sum([v for task, v in self.results.items() if v is not None and task in mc_tasks]) / len(mc_tasks)
+
+        # average = sum([(v-baselines.get(task,0))/(100-baselines.get(task,0))*100 for task, v in self.results.items() if v is not None and task in all_tasks]) / len(all_tasks)
+        # average_g = sum([(v-baselines.get(task,0))/(100-baselines.get(task,0))*100 for task, v in self.results.items() if v is not None and task in g_tasks]) / len(g_tasks)
+        # average_mc = sum([(v-baselines.get(task,0))/(100-baselines.get(task,0))*100 for task, v in self.results.items() if v is not None and task in mc_tasks]) / len(mc_tasks)
+
+        data_dict = {}
         # data_dict = {
         #     "eval_name": self.eval_name,  # not a column, just a save name,
         #     AutoEvalColumn.precision.name: self.precision.value.name,
@@ -188,7 +202,6 @@ class EvalResult:
         except KeyError:
             print(f"Could not find model type")
 
-
         try:
             data_dict[AutoEvalColumn.model_type_symbol.name] = self.model_type.value.symbol
         except KeyError:
@@ -209,7 +222,8 @@ class EvalResult:
             print(f"AttributeError architecture")
 
         try:
-            data_dict[AutoEvalColumn.model.name] = make_clickable_model(self.full_model) if self.still_on_hub else self.full_model
+            data_dict[AutoEvalColumn.model.name] = make_clickable_model(
+                self.full_model) if self.still_on_hub else self.full_model
         except KeyError:
             print(f"Could not find model")
 
@@ -305,8 +319,8 @@ def get_request_file_for_model(requests_path, model_name, precision):
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
             if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
+                    req_content["status"] in ["FINISHED"]
+                    and req_content["precision"] == precision.split(".")[-1]
             ):
                 request_file = tmp_request_file
     return request_file
@@ -330,30 +344,48 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
+    # print('PATHS:', model_result_filepaths)
+
     eval_results = {}
-    for n_shot in [0,5]:
+    for n_shot in [0, 5]:
         for model_result_filepath in model_result_filepaths:
             # Creation of result
             eval_result = EvalResult.init_from_json_file(model_result_filepath, n_shot_num=n_shot)
             eval_result.update_with_request_file(requests_path)
-            #update with metadata
+            # update with metadata
             eval_result.update_with_metadata(metadata)
 
-
             # Store results of same eval together
             eval_name = f"{eval_result.eval_name}_{n_shot}-shot"
             if eval_name in eval_results.keys():
-                eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-                #TODO: log updated
+
+                for k, (v, start_date) in eval_result.results.items():
+                    if v is not None:
+                        if k in eval_results[eval_name].results:
+                            if start_date > eval_results[eval_name].results[k][1]:
+                                print(
+                                    f"Overwriting {eval_name}.results {k} {eval_results[eval_name].results[k]} with {v}: {model_result_filepath} {n_shot} {eval_result.start_date}  {eval_results[eval_name].start_date}")
+                                eval_results[eval_name].results[k] = (v, start_date)
+                            else:
+                                print(
+                                    f"Skipping {eval_name} {eval_result.start_date} {eval_results[eval_name].start_date}: {model_result_filepath} {n_shot}")
+                        else:
+                            eval_results[eval_name].results[k] = (v, start_date)
+                # eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+                # TODO: log updated
+
             else:
                 eval_results[eval_name] = eval_result
 
+    for k,v in eval_results.items():
+        v.results = {k: v for k, (v, start_date) in v.results.items()}
+
     results = []
     for v in eval_results.values():
         try:
             print(v)
-            v.to_dict() # we test if the dict version is complete
-            #if v.results:
+            v.to_dict()  # we test if the dict version is complete
+            # if v.results:
             results.append(v)
         except KeyError:  # not all eval values present
             print(f"not all eval values present {v.eval_name} {v.full_model}")
@@ -370,7 +402,7 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
                     missing_results_for_task[task_name].append(f"{v.full_model}|{v.org_and_model}")
                 else:
                     missing_results_for_task[task_name] = [f"{v.full_model}|{v.org_and_model}"]
-        if r[AutoEvalColumn.lang.name] is None or r[AutoEvalColumn.lang.name]=="?":
+        if r[AutoEvalColumn.lang.name] is None or r[AutoEvalColumn.lang.name] == "?":
             missing_metadata.append(f"{v.full_model}")
 
     # print('missing_results_for_task', missing_results_for_task)
@@ -386,5 +418,4 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
         print(model)
     print()
 
-
     return results