test

src/about.py

@@ -12,8 +12,8 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task0 = Task("speed", "spq", "Speed")
+    task1 = Task("contamination", "score", "Contamination")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/leaderboard/read_evals.py

@@ -60,6 +60,10 @@ class EvalResult:
         still_on_hub, _, model_config = is_model_on_hub(
             full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
         )
+        
+        ##make still_on_hub always true for now:
+        still_on_hub = True
+
         architecture = "?"
         if model_config is not None:
             architectures = getattr(model_config, "architectures", None)
@@ -90,11 +94,10 @@ class EvalResult:
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
         request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
-            print(request_file)
+            
             self.model_type = ModelType.from_str(request.get("model_type", ""))
             self.weight_type = WeightType[request.get("weight_type", "Original")]
             self.license = request.get("license", "?")
@@ -106,7 +109,7 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        average = self.results["average"]
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             AutoEvalColumn.precision.name: self.precision.value.name,
@@ -135,6 +138,7 @@ def get_request_file_for_model(requests_path, model_name, precision):
         requests_path,
         f"{model_name}_eval_request_*.json",
     )
+    
     request_files = glob.glob(request_files)
 
     # Select correct request file (precision)
@@ -174,7 +178,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
         eval_result.update_with_request_file(requests_path)
-        
+
         # Store results of same eval together
         eval_name = eval_result.eval_name
         if eval_name in eval_results.keys():
@@ -183,13 +187,17 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             eval_results[eval_name] = eval_result
 
     results = []
-    
+    #print(eval_results.values())
     for v in eval_results.values():
         try:
             print(v.to_dict())
             v.to_dict() # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
+            print("Key error in eval result, skipping")
+            
+            print(v)
+            print(v.to_dict())
             continue
 
     print(results)