Update functions.py

functions.py

@@ -16,128 +16,95 @@ finished_models = get_datas(data)
 df = pd.DataFrame(finished_models)
 
 desc = """
-This is an automated PR created with https://huggingface.co/spaces/eduagarcia-temp/portuguese-leaderboard-results-to-modelcard
-
-The purpose of this PR is to add evaluation results from the Open Portuguese LLM Leaderboard to your model card.
-
-If you encounter any issues, please report them to https://huggingface.co/spaces/eduagarcia-temp/portuguese-leaderboard-results-to-modelcard/discussions
+This is an automated PR created with https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr
+The purpose of this PR is to add evaluation results from the Open LLM Leaderboard to your model card.
+If you encounter any issues, please report them to https://huggingface.co/spaces/Weyaxi/open-llm-leaderboard-results-pr/discussions
 """
 
 def search(df, value):
-    result_df = df[df["Model Name"] == value]
+    result_df = df[df["Model"] == value]
     return result_df.iloc[0].to_dict() if not result_df.empty else None
 
 
 def get_details_url(repo):
-   #author, model = repo.split("/")
-   return f"https://huggingface.co/datasets/eduagarcia-temp/llm_pt_leaderboard_raw_results/tree/main/{repo}"
+   author, model = repo.split("/")
+   return f"https://huggingface.co/datasets/open-llm-leaderboard/details_{author}__{model}"
 
 
 def get_query_url(repo):
-  return f"https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard?query={repo}"
+  return f"https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query={repo}"
 
 
 def get_task_summary(results):
   return {
-      "ENEM":
-          {"dataset_type":"eduagarcia/enem_challenge",
-          "dataset_name":"ENEM Challenge (No Images)",
-          "metric_type":"acc",
-          "metric_value":results["ENEM"],
-          "dataset_config": None,
-          "dataset_split":"train",
-          "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 3},
-          "metric_name":"accuracy"
-          },
-      "BLUEX":
-          {"dataset_type":"eduagarcia-temp/BLUEX_without_images",
-          "dataset_name":"BLUEX (No Images)",
-          "metric_type":"acc",
-          "metric_value":results["BLUEX"],
-          "dataset_config": None,
-          "dataset_split":"train",
-          "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 3},
-          "metric_name":"accuracy"
-          },
-      "OAB Exams":
-          {"dataset_type":"eduagarcia/oab_exams",
-          "dataset_name":"OAB Exams",
-          "metric_type":"acc",
-          "metric_value":results["OAB Exams"],
-          "dataset_config": None,
-          "dataset_split":"train",
-          "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 3},
-          "metric_name":"accuracy"
-          },
-      "ASSIN2 RTE":
-          {"dataset_type":"assin2",
-          "dataset_name":"Assin2 RTE",
-          "metric_type":"f1_macro",
-          "metric_value":results["ASSIN2 RTE"],
-          "dataset_config": None,
-          "dataset_split":"test",
-          "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 15},
-          "metric_name":"f1-macro"
-          },
-      "ASSIN2 STS":
-          {"dataset_type":"assin2",
-          "dataset_name":"Assin2 STS",
-          "metric_type":"pearson",
-          "metric_value":results["ASSIN2 STS"],
-          "dataset_config": None,
+      "ARC":
+          {"dataset_type":"ai2_arc",
+          "dataset_name":"AI2 Reasoning Challenge (25-Shot)",
+          "metric_type":"acc_norm",
+          "metric_value":results["ARC"],
+          "dataset_config":"ARC-Challenge",
           "dataset_split":"test",
           "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 15},
-          "metric_name":"pearson"
+          "dataset_args":{"num_few_shot": 25},
+          "metric_name":"normalized accuracy"
           },
-      "FAQUAD NLI":
-          {"dataset_type":"ruanchaves/faquad-nli",
-          "dataset_name":"FaQuAD NLI",
-          "metric_type":"f1_macro",
-          "metric_value":results["FAQUAD NLI"],
-          "dataset_config": None,
-          "dataset_split":"test",
+      "HellaSwag":
+          {"dataset_type":"hellaswag",
+          "dataset_name":"HellaSwag (10-Shot)",
+          "metric_type":"acc_norm",
+          "metric_value":results["HellaSwag"],
+          "dataset_config":None,
+          "dataset_split":"validation",
           "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 15},
-          "metric_name":"f1-macro"
+          "dataset_args":{"num_few_shot": 10},
+          "metric_name":"normalized accuracy"
           },
-      "HateBR":
-          {"dataset_type":"eduagarcia/portuguese_benchmark",
-          "dataset_name":"HateBR Binary",
-          "metric_type":"f1_macro",
-          "metric_value":results["HateBR"],
-          "dataset_config": None,
+      "MMLU":
+      {
+          "dataset_type":"cais/mmlu",
+          "dataset_name":"MMLU (5-Shot)",
+          "metric_type":"acc",
+          "metric_value":results["MMLU"],
+          "dataset_config":"all",
           "dataset_split":"test",
           "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 25},
-          "metric_name":"f1-macro"
-          },
-      "PT Hate Speech":
-          {"dataset_type":"eduagarcia/portuguese_benchmark",
-          "dataset_name":"PT Hate Speech Binary",
-          "metric_type":"f1_macro",
-          "metric_value":results["PT Hate Speech"],
-          "dataset_config": None,
-          "dataset_split":"test",
+          "dataset_args":{"num_few_shot": 5},
+          "metric_name":"accuracy"
+      },
+      "TruthfulQA":
+      {
+          "dataset_type":"truthful_qa",
+          "dataset_name":"TruthfulQA (0-shot)",
+          "metric_type":"mc2",
+          "metric_value":results["TruthfulQA"],
+          "dataset_config":"multiple_choice",
+          "dataset_split":"validation",
           "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 25},
-          "metric_name":"f1-macro"
-          },
-      "tweetSentBR":
-          {"dataset_type":"eduagarcia-temp/tweetsentbr",
-          "dataset_name":"tweetSentBR",
-          "metric_type":"f1_macro",
-          "metric_value":results["tweetSentBR"],
-          "dataset_config": None,
+          "dataset_args":{"num_few_shot": 0},
+          "metric_name":None
+      },
+      "Winogrande":
+      {
+          "dataset_type":"winogrande",
+          "dataset_name":"Winogrande (5-shot)",
+          "metric_type":"acc",
+          "metric_value":results["Winogrande"],
+          "dataset_config":"winogrande_xl",
+          "dataset_split":"validation",
+          "dataset_args":{"num_few_shot": 5},
+          "metric_name":"accuracy"
+      },
+      "GSM8K":
+      {
+          "dataset_type":"gsm8k",
+          "dataset_name":"GSM8k (5-shot)",
+          "metric_type":"acc",
+          "metric_value":results["GSM8K"],
+          "dataset_config":"main",
           "dataset_split":"test",
-          "dataset_revision":None,
-          "dataset_args":{"num_few_shot": 25},
-          "metric_name":"f1-macro"
-          }
+          "dataset_args":{"num_few_shot": 5},
+          "metric_name":"accuracy"
+      }
   }
 
 
@@ -147,12 +114,11 @@ def get_eval_results(repo):
   task_summary = get_task_summary(results)
   md_writer = MarkdownTableWriter()
   md_writer.headers = ["Metric", "Value"]
-  md_writer.value_matrix = [["Average", f"**{results['Average ⬆️']}**"]] + [[v["dataset_name"], v["metric_value"]] for v in task_summary.values()]
+  md_writer.value_matrix = [["Avg.", results['Average ⬆️']]] + [[v["dataset_name"], v["metric_value"]] for v in task_summary.values()]
 
   text = f"""
-# [Open Portuguese LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/eduagarcia/open_pt_llm_leaderboard)
+# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
 Detailed results can be found [here]({get_details_url(repo)})
-
 {md_writer.dumps()}
 """
   return text