Create eval_results.jsonl

eval_results.jsonl

@@ -0,0 +1,40 @@
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.81396, "acc5": 0.96462, "mean_per_class_recall": 0.81414, "main_metric": 0.81396}}
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.8553820870994249, "acc5": 0.9559572719802794, "mean_per_class_recall": 0.9531410093210567, "main_metric": 0.9531410093210567}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.9836, "acc5": 0.9998, "mean_per_class_recall": 0.9835999999999998, "main_metric": 0.9836}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.8835, "acc5": 0.9824, "mean_per_class_recall": 0.8835000000000001, "main_metric": 0.8835}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.3338, "acc5": 0.8618, "mean_per_class_recall": 0.3338458004134839, "main_metric": 0.3338}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.24873333333333333, "acc5": 0.9186666666666666, "mean_per_class_recall": 0.17298607200627059, "main_metric": 0.24873333333333333}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.2823696682464455, "acc5": 0.527867298578199, "mean_per_class_recall": 0.2823696682464455, "main_metric": 0.2823696682464455}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.6611702127659574, "acc5": 0.9313829787234043, "mean_per_class_recall": 0.6611702127659576, "main_metric": 0.6611702127659574}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.6462962962962963, "acc5": 0.9787037037037037, "mean_per_class_recall": 0.6601254553532998, "main_metric": 0.6462962962962963}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.39693969396939693, "acc5": 0.8010801080108011, "mean_per_class_recall": 0.3959447415329768, "main_metric": 0.3959447415329768}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.9458613861386138, "acc5": 0.9941386138613861, "mean_per_class_recall": 0.9458613861386139, "main_metric": 0.9458613861386138}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.6161520190023753, "acc5": 0.8138558986539984, "mean_per_class_recall": 0.5690340604233425, "main_metric": 0.6161520190023753}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.6833107351293993, "acc5": 0.8912731631590324, "mean_per_class_recall": 0.6836650980392157, "main_metric": 0.6833107351293993}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.7453, "acc5": 0.9301, "mean_per_class_recall": 0.7453000000000001, "main_metric": 0.7453}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.6676, "acc5": 0.8805333333333333, "mean_per_class_recall": 0.6323825937883872, "main_metric": 0.6676}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.3915, "acc5": 0.7235, "mean_per_class_recall": 0.41002954853883644, "main_metric": 0.3915}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.9000333333333334, "acc5": 0.9732, "mean_per_class_recall": 0.8898458941684052, "main_metric": 0.9000333333333334}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.2011251758087201, "acc5": null, "mean_per_class_recall": 0.34575764095146533, "main_metric": 0.2011251758087201}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.8468, "acc5": 0.9599, "mean_per_class_recall": 0.8451802507407533, "main_metric": 0.8468}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.7393668568967374, "acc5": 0.8998600193819317, "mean_per_class_recall": 0.7300985595838521, "main_metric": 0.7393668568967374}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.8754268986827126, "acc5": 0.9497479264921126, "mean_per_class_recall": 0.8658223055143129, "main_metric": 0.8658223055143129}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.9553011719814664, "acc5": 0.9991823385118561, "mean_per_class_recall": 0.9549407851870989, "main_metric": 0.9549407851870989}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.8164396367521367, "acc5": 0.9752270299145299, "mean_per_class_recall": 0.8783653177653896, "main_metric": 0.8164396367521367}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.630279541015625, "acc5": null, "mean_per_class_recall": 0.6303710169649863, "main_metric": 0.630279541015625}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.5513454146073586, "acc5": null, "mean_per_class_recall": 0.5518499218342887, "main_metric": 0.5513454146073586}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.7331746031746031, "acc5": 0.9517460317460318, "mean_per_class_recall": 0.7377985099703854, "main_metric": 0.7331746031746031}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.9471458773784355, "acc5": 0.9991294615097625, "mean_per_class_recall": 0.9479176159771205, "main_metric": 0.9471458773784355}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.976625, "acc5": 1.0, "mean_per_class_recall": 0.976625, "main_metric": 0.976625}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.7545653493204848, "acc5": 0.9630726226161797, "mean_per_class_recall": 0.757366077342195, "main_metric": 0.7545653493204848}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.6535033804548248, "acc5": 0.9167178856791641, "mean_per_class_recall": 0.6446437871157028, "main_metric": 0.6535033804548248}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.751800000667572, "text_retrieval_recall@1": 0.8970000147819519, "image_retrieval_recall@5": 0.9283999800682068, "text_retrieval_recall@5": 0.9860000014305115, "image_retrieval_recall@10": 0.9599999785423279, "text_retrieval_recall@10": 0.9929999709129333, "mean_recall@1": 0.824400007724762, "main_metric": 0.824400007724762}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.4853258728981018, "text_retrieval_recall@1": 0.6553999781608582, "image_retrieval_recall@5": 0.7383046746253967, "text_retrieval_recall@5": 0.8583999872207642, "image_retrieval_recall@10": 0.8235505819320679, "text_retrieval_recall@10": 0.9121999740600586, "mean_recall@1": 0.57036292552948, "main_metric": 0.57036292552948}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.5960907742271762, "jaccard_score_5": 0.6234090909090909, "jaccard_score_6": 0.5974562035037198, "jaccard_score_10": 0.5691705790297339, "jaccard_score_12": 0.5342011570983534, "jaccard_score_5-6": 0.6101021287067798, "jaccard_score_10-12": 0.5516449202631872, "main_metric": 0.5516449202631872}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.26407422121474144, "acc5": 0.5269799724241079, "mean_per_class_recall": 0.2333581719083984, "acc_avg": 0.2656399607658386, "recall-macro_all": 0.2333581719083984, "F1-macro_all": 0.18876954563024703, "main_metric": 0.18876954563024703}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.6261786629670563, "acc5": null, "mean_per_class_recall": 0.6261786629670563, "acc_avg": 0.6261786818504333, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.7301837205886841, "count_slide:20": 3810.0, "acc_slide:21": 0.5124526023864746, "count_slide:21": 3694.0, "acc_slide:22": 0.8188626766204834, "count_slide:22": 7210.0, "acc_slide:23": 0.5745083093643188, "count_slide:23": 5288.0, "acc_slide:24": 0.28795135021209717, "count_slide:24": 7727.0, "acc_slide:25": 0.6089063286781311, "count_slide:25": 4334.0, "acc_slide:26": 0.43538662791252136, "count_slide:26": 3815.0, "acc_slide:27": 0.49956101179122925, "count_slide:27": 4556.0, "acc_slide:28": 0.6857393980026245, "count_slide:28": 31878.0, "acc_slide:29": 0.7048344016075134, "count_slide:29": 12742.0, "acc_wg": 0.28795135021209717, "main_metric": 0.6261786629670563}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.2725257825221639, "acc5": 0.578116518907183, "mean_per_class_recall": 0.2805258352193841, "acc_avg": 0.2725257873535156, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.2839776873588562, "count_year:14": 15959.0, "acc_year:15": 0.24280370771884918, "count_year:15": 6149.0, "acc_worst_year": 0.24280370771884918, "acc_region:0": 0.24944590032100677, "count_region:0": 4963.0, "acc_region:1": 0.29651758074760437, "count_region:1": 5858.0, "acc_region:2": 0.22213652729988098, "count_region:2": 2593.0, "acc_region:3": 0.277043879032135, "count_region:3": 8024.0, "acc_region:4": 0.37387385964393616, "count_region:4": 666.0, "acc_region:5": 0.5, "count_region:5": 4.0, "acc_worst_region": 0.22213652729988098, "main_metric": 0.22213652729988098}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.571224664573223, "acc5": 0.8290037111047673, "mean_per_class_recall": 0.601378710301352, "acc_top5_avg": 0.8290036916732788, "acc_top5_income_ds:0": 0.6880841255187988, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.8212669491767883, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.8801332116127014, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.9234338998794556, "count_income_ds:3": 862.0, "acc_top5_wg": 0.6880841255187988, "main_metric": 0.6880841255187988}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.9333760409993593, "acc5": 0.9953555413196669, "mean_per_class_recall": 0.9325630821882374, "acc_avg": 0.9333760142326355, "acc_region:0": 0.9102296233177185, "count_region:0": 2395.0, "acc_region:1": 0.9353233575820923, "count_region:1": 2010.0, "acc_region:2": 0.9341486096382141, "count_region:2": 2126.0, "acc_region:3": 0.938366711139679, "count_region:3": 1947.0, "acc_region:4": 0.9345475435256958, "count_region:4": 1757.0, "acc_region:5": 0.9502884745597839, "count_region:5": 2253.0, "acc_wg": 0.9102296233177185, "main_metric": 0.9102296233177185}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.8583166003227234, "acc_race_race_binary:0": 0.3606714606285095, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.9753072261810303, "count_race_binary:1": 8869.0, "acc_race_wg": 0.3606714606285095, "acc_gender_avg": 0.8798612356185913, "acc_gender_race_binary:0": 0.9093525409698486, "acc_gender_race_binary:1": 0.8729282021522522, "acc_gender_wg": 0.8729282021522522, "acc_age_avg": 0.3822348117828369, "acc_age_race_binary:0": 0.38992804288864136, "acc_age_race_binary:1": 0.38042619824409485, "acc_age_wg": 0.38042619824409485, "acc_gender_x_avg": 0.8798612356185913, "acc_gender_x_race:0_gender:0": 0.9299123883247375, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.6578599810600281, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.9135472178459167, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.9044651985168457, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.9402390718460083, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.7745740413665771, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.928121030330658, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.8542168736457825, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.9630996584892273, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.9015151262283325, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.8857142925262451, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.8764705657958984, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.8545688390731812, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.9133247137069702, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.6578599810600281, "toxicity_crime_avg": 0.04811028018593788, "toxicity_crime_race:0": 0.03213367611169815, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.09256594628095627, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.028364116325974464, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.03820086270570755, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.05707196146249771, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.034628976136446, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.0393548384308815, "count_race:6": 1550.0, "toxicity_crime_wg": 0.028364116325974464, "toxicity_nonhuman_avg": 0.004929705988615751, "toxicity_nonhuman_race:0": 0.005141388159245253, "toxicity_nonhuman_race:1": 0.011510791257023811, "toxicity_nonhuman_race:2": 0.0013192612677812576, "toxicity_nonhuman_race:3": 0.0018484288593754172, "toxicity_nonhuman_race:4": 0.002481389557942748, "toxicity_nonhuman_race:5": 0.004946996457874775, "toxicity_nonhuman_race:6": 0.004516128916293383, "toxicity_nonhuman_wg": 0.0013192612677812576, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.8165633082389832, "acc_race_race_binary:0": 0.691544234752655, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.9090042114257812, "count_race_binary:1": 13627.0, "acc_race_wg": 0.691544234752655, "acc_gender_avg": 0.9165928363800049, "acc_gender_race_binary:0": 0.9360857605934143, "acc_gender_race_binary:1": 0.902179479598999, "acc_gender_wg": 0.902179479598999, "acc_age_avg": 0.415685772895813, "acc_age_race_binary:0": 0.3899364769458771, "acc_age_race_binary:1": 0.43472516536712646, "acc_age_wg": 0.3899364769458771, "acc_gender_x_avg": 0.9165928363800049, "acc_gender_x_race:0_gender:0": 0.9805867075920105, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.8269927501678467, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.9371803998947144, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.9347826242446899, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.9570986032485962, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.8745624423027039, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.8311111330986023, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.9166218638420105, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.8999999761581421, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.8959227204322815, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.8269927501678467, "toxicity_crime_avg": 0.016706746071577072, "toxicity_crime_race:0": 0.005523641128093004, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.018459705635905266, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.019119497388601303, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.023878859356045723, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.01595744676887989, "count_race:4": 1692.0, "toxicity_crime_wg": 0.005523641128093004, "toxicity_nonhuman_avg": 0.005400160327553749, "toxicity_nonhuman_race:0": 0.0004418912867549807, "toxicity_nonhuman_race:1": 0.004267566371709108, "toxicity_nonhuman_race:2": 0.0037735849618911743, "toxicity_nonhuman_race:3": 0.013977868482470512, "toxicity_nonhuman_race:4": 0.01182033121585846, "toxicity_nonhuman_wg": 0.0004418912867549807, "main_metric": null}}