missing

src/leaderboard/read_evals.py

@@ -456,15 +456,26 @@ def get_raw_eval_results(results_path: str, requests_path: str, metadata) -> lis
                 if task_name in missing_results_for_task:
                     missing_results_for_task[task_name].append(f"{v.full_model}|{v.org_and_model}")
                     if v.still_on_hub and task.value.benchmark in all_tasks:
-                        print(f'batch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
+                        for_run.append([r["n_shot"], task.value.benchmark, v.full_model])
+                        # print(f'sbatch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
                 else:
                     missing_results_for_task[task_name] = [f"{v.full_model}|{v.org_and_model}"]
                     if v.still_on_hub and task.value.benchmark in all_tasks:
-                        print(f'batch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
+                        for_run.append([r["n_shot"], task.value.benchmark, v.full_model])
+                        # print(f'sbatch start.sh "bash eval_model_task_bs1.sh {r["n_shot"]} {task.value.benchmark} {v.full_model}"')
         if r[AutoEvalColumn.lang.name] is None or r[AutoEvalColumn.lang.name] == "?":
             missing_metadata.append(f"{v.full_model}")
         all_models.append((v.full_model, v.num_params, v.still_on_hub))
 
+    print(f"Missing sbatch results:")
+    for r in for_run:
+        fm=r[2].replace(',multiturn','')
+        if ',chat' in fm:
+            fm=fm.replace(',chat','')
+            print(f'sbatch start.sh "bash eval_model_task_bs1_chat.sh {r[0]} {r[1]} {fm}"')
+        else:
+            print(f'sbatch start.sh "bash eval_model_task_bs1.sh {r[0]} {r[1]} {fm}"')
+
     # print('missing_results_for_task', missing_results_for_task)
     for task, models in missing_results_for_task.items():
         print(f"Missing results for {task} for {len(models)} models")