Upload llm-eval-Kiwi-1.0-0.7B-32k-Instruct-complete.json

llm-eval-Kiwi-1.0-0.7B-32k-Instruct-complete.json

@@ -0,0 +1,207 @@
+{
+    "arc_challenge": {
+        "alias": "arc_challenge",
+        "acc,none": 0.32081911262798635,
+        "acc_stderr,none": 0.013640943091946526,
+        "acc_norm,none": 0.32337883959044367,
+        "acc_norm_stderr,none": 0.013669421630012118
+    },
+    "gpqa_diamond_cot_n_shot": {
+        "alias": "gpqa_diamond_cot_n_shot",
+        "exact_match,strict-match": 0.005050505050505051,
+        "exact_match_stderr,strict-match": 0.005050505050505052,
+        "exact_match,flexible-extract": 0.23232323232323232,
+        "exact_match_stderr,flexible-extract": 0.03008862949021749
+    },
+    "gpqa_diamond_cot_zeroshot": {
+        "alias": "gpqa_diamond_cot_zeroshot",
+        "exact_match,strict-match": 0.0,
+        "exact_match_stderr,strict-match": 0.0,
+        "exact_match,flexible-extract": 0.1717171717171717,
+        "exact_match_stderr,flexible-extract": 0.026869716187429917
+    },
+    "gpqa_diamond_generative_n_shot": {
+        "alias": "gpqa_diamond_generative_n_shot",
+        "exact_match,strict-match": 0.0,
+        "exact_match_stderr,strict-match": 0.0,
+        "exact_match,flexible-extract": 0.2222222222222222,
+        "exact_match_stderr,flexible-extract": 0.02962022787479049
+    },
+    "gpqa_diamond_n_shot": {
+        "alias": "gpqa_diamond_n_shot",
+        "acc,none": 0.2878787878787879,
+        "acc_stderr,none": 0.03225883512300993,
+        "acc_norm,none": 0.2878787878787879,
+        "acc_norm_stderr,none": 0.03225883512300993
+    },
+    "gpqa_diamond_zeroshot": {
+        "alias": "gpqa_diamond_zeroshot",
+        "acc,none": 0.3181818181818182,
+        "acc_stderr,none": 0.03318477333845331,
+        "acc_norm,none": 0.3181818181818182,
+        "acc_norm_stderr,none": 0.03318477333845331
+    },
+    "gpqa_extended_cot_n_shot": {
+        "alias": "gpqa_extended_cot_n_shot",
+        "exact_match,strict-match": 0.005494505494505495,
+        "exact_match_stderr,strict-match": 0.0031664282264934744,
+        "exact_match,flexible-extract": 0.21062271062271062,
+        "exact_match_stderr,flexible-extract": 0.017466128919599165
+    },
+    "gpqa_extended_cot_zeroshot": {
+        "alias": "gpqa_extended_cot_zeroshot",
+        "exact_match,strict-match": 0.0,
+        "exact_match_stderr,strict-match": 0.0,
+        "exact_match,flexible-extract": 0.2032967032967033,
+        "exact_match_stderr,flexible-extract": 0.017239125210643556
+    },
+    "gpqa_extended_generative_n_shot": {
+        "alias": "gpqa_extended_generative_n_shot",
+        "exact_match,strict-match": 0.0,
+        "exact_match_stderr,strict-match": 0.0,
+        "exact_match,flexible-extract": 0.20695970695970695,
+        "exact_match_stderr,flexible-extract": 0.017353707620392123
+    },
+    "gpqa_extended_n_shot": {
+        "alias": "gpqa_extended_n_shot",
+        "acc,none": 0.25824175824175827,
+        "acc_stderr,none": 0.01874762138022981,
+        "acc_norm,none": 0.25824175824175827,
+        "acc_norm_stderr,none": 0.01874762138022981
+    },
+    "gpqa_extended_zeroshot": {
+        "alias": "gpqa_extended_zeroshot",
+        "acc,none": 0.26556776556776557,
+        "acc_stderr,none": 0.018917567557968248,
+        "acc_norm,none": 0.26556776556776557,
+        "acc_norm_stderr,none": 0.018917567557968248
+    },
+    "gpqa_main_cot_n_shot": {
+        "alias": "gpqa_main_cot_n_shot",
+        "exact_match,strict-match": 0.004464285714285714,
+        "exact_match_stderr,strict-match": 0.003153193710465662,
+        "exact_match,flexible-extract": 0.20535714285714285,
+        "exact_match_stderr,flexible-extract": 0.019106763899193864
+    },
+    "gpqa_main_cot_zeroshot": {
+        "alias": "gpqa_main_cot_zeroshot",
+        "exact_match,strict-match": 0.002232142857142857,
+        "exact_match_stderr,strict-match": 0.0022321428571428744,
+        "exact_match,flexible-extract": 0.20089285714285715,
+        "exact_match_stderr,flexible-extract": 0.018950950292372627
+    },
+    "gpqa_main_generative_n_shot": {
+        "alias": "gpqa_main_generative_n_shot",
+        "exact_match,strict-match": 0.0,
+        "exact_match_stderr,strict-match": 0.0,
+        "exact_match,flexible-extract": 0.24330357142857142,
+        "exact_match_stderr,flexible-extract": 0.020294638625866775
+    },
+    "gpqa_main_n_shot": {
+        "alias": "gpqa_main_n_shot",
+        "acc,none": 0.328125,
+        "acc_stderr,none": 0.0222080353262888,
+        "acc_norm,none": 0.328125,
+        "acc_norm_stderr,none": 0.0222080353262888
+    },
+    "gpqa_main_zeroshot": {
+        "alias": "gpqa_main_zeroshot",
+        "acc,none": 0.234375,
+        "acc_stderr,none": 0.020035949758324928,
+        "acc_norm,none": 0.234375,
+        "acc_norm_stderr,none": 0.020035949758324928
+    },
+    "hellaswag": {
+        "alias": "hellaswag",
+        "acc,none": 0.3990240987851026,
+        "acc_stderr,none": 0.004886969266944274,
+        "acc_norm,none": 0.4858593905596495,
+        "acc_norm_stderr,none": 0.004987785530475671
+    },
+    "ifeval": {
+        "alias": "ifeval",
+        "prompt_level_strict_acc,none": 0.1423290203327172,
+        "prompt_level_strict_acc_stderr,none": 0.0150352345622028,
+        "inst_level_strict_acc,none": 0.23381294964028776,
+        "inst_level_strict_acc_stderr,none": "N/A",
+        "prompt_level_loose_acc,none": 0.16820702402957485,
+        "prompt_level_loose_acc_stderr,none": 0.016096550188063007,
+        "inst_level_loose_acc,none": 0.2709832134292566,
+        "inst_level_loose_acc_stderr,none": "N/A"
+    },
+    "mmlu_pro": {
+        "exact_match,custom-extract": 0.12890625,
+        "exact_match_stderr,custom-extract": 0.0030434275267919382,
+        "alias": "mmlu_pro"
+    },
+    "mmlu_pro_biology": {
+        "alias": " - biology",
+        "exact_match,custom-extract": 0.17154811715481172,
+        "exact_match_stderr,custom-extract": 0.014088673719424991
+    },
+    "mmlu_pro_business": {
+        "alias": " - business",
+        "exact_match,custom-extract": 0.11913814955640051,
+        "exact_match_stderr,custom-extract": 0.011540276571470746
+    },
+    "mmlu_pro_chemistry": {
+        "alias": " - chemistry",
+        "exact_match,custom-extract": 0.09187279151943463,
+        "exact_match_stderr,custom-extract": 0.008588859484723885
+    },
+    "mmlu_pro_computer_science": {
+        "alias": " - computer_science",
+        "exact_match,custom-extract": 0.12926829268292683,
+        "exact_match_stderr,custom-extract": 0.016589241600938224
+    },
+    "mmlu_pro_economics": {
+        "alias": " - economics",
+        "exact_match,custom-extract": 0.18838862559241706,
+        "exact_match_stderr,custom-extract": 0.013467519528615376
+    },
+    "mmlu_pro_engineering": {
+        "alias": " - engineering",
+        "exact_match,custom-extract": 0.08565531475748193,
+        "exact_match_stderr,custom-extract": 0.008994860895662203
+    },
+    "mmlu_pro_health": {
+        "alias": " - health",
+        "exact_match,custom-extract": 0.14425427872860636,
+        "exact_match_stderr,custom-extract": 0.012292088876836848
+    },
+    "mmlu_pro_history": {
+        "alias": " - history",
+        "exact_match,custom-extract": 0.15485564304461943,
+        "exact_match_stderr,custom-extract": 0.018558256274560196
+    },
+    "mmlu_pro_law": {
+        "alias": " - law",
+        "exact_match,custom-extract": 0.12534059945504086,
+        "exact_match_stderr,custom-extract": 0.009983182840215515
+    },
+    "mmlu_pro_math": {
+        "alias": " - math",
+        "exact_match,custom-extract": 0.09844559585492228,
+        "exact_match_stderr,custom-extract": 0.008108251878622208
+    },
+    "mmlu_pro_other": {
+        "alias": " - other",
+        "exact_match,custom-extract": 0.14502164502164502,
+        "exact_match_stderr,custom-extract": 0.011590258522971733
+    },
+    "mmlu_pro_philosophy": {
+        "alias": " - philosophy",
+        "exact_match,custom-extract": 0.14829659318637275,
+        "exact_match_stderr,custom-extract": 0.015925574493977492
+    },
+    "mmlu_pro_physics": {
+        "alias": " - physics",
+        "exact_match,custom-extract": 0.1100846805234796,
+        "exact_match_stderr,custom-extract": 0.008687612438998104
+    },
+    "mmlu_pro_psychology": {
+        "alias": " - psychology",
+        "exact_match,custom-extract": 0.17042606516290726,
+        "exact_match_stderr,custom-extract": 0.013318839850041711
+    }
+}