| { | |
| "metadata": { | |
| "timestamp": "2025-03-13T15:23:57.734843", | |
| "vidore_benchmark_version": "5.0.1.dev5+g1548c2d.d20250313" | |
| }, | |
| "metrics": { | |
| "vidore/restaurant_esg_reports_beir": { | |
| "ndcg_at_1": 0.48718, | |
| "ndcg_at_3": 0.51987, | |
| "ndcg_at_5": 0.53707, | |
| "ndcg_at_10": 0.58691, | |
| "ndcg_at_20": 0.60769, | |
| "ndcg_at_50": 0.6397, | |
| "ndcg_at_100": 0.65482, | |
| "map_at_1": 0.36186, | |
| "map_at_3": 0.4602, | |
| "map_at_5": 0.48616, | |
| "map_at_10": 0.51598, | |
| "map_at_20": 0.52449, | |
| "map_at_50": 0.53241, | |
| "map_at_100": 0.5362, | |
| "recall_at_1": 0.36186, | |
| "recall_at_3": 0.52821, | |
| "recall_at_5": 0.59359, | |
| "recall_at_10": 0.70563, | |
| "recall_at_20": 0.76315, | |
| "recall_at_50": 0.88615, | |
| "recall_at_100": 0.94866, | |
| "precision_at_1": 0.5, | |
| "precision_at_3": 0.27564, | |
| "precision_at_5": 0.19231, | |
| "precision_at_10": 0.13077, | |
| "precision_at_20": 0.07596, | |
| "precision_at_50": 0.03808, | |
| "precision_at_100": 0.02192, | |
| "mrr_at_1": 0.5, | |
| "mrr_at_3": 0.5769230769230769, | |
| "mrr_at_5": 0.5865384615384615, | |
| "mrr_at_10": 0.6023733211233211, | |
| "mrr_at_20": 0.6069520757020757, | |
| "mrr_at_50": 0.6103908065114961, | |
| "mrr_at_100": 0.6106960568167464, | |
| "naucs_at_1_max": 0.015424312502453706, | |
| "naucs_at_1_std": -0.13924671268887046, | |
| "naucs_at_1_diff1": 0.774635283388261, | |
| "naucs_at_3_max": 0.08232812603721668, | |
| "naucs_at_3_std": 0.14582513042567308, | |
| "naucs_at_3_diff1": 0.3413749493398797, | |
| "naucs_at_5_max": 0.04733940764311593, | |
| "naucs_at_5_std": 0.201486754820981, | |
| "naucs_at_5_diff1": 0.1679332256651539, | |
| "naucs_at_10_max": 0.15213696092763468, | |
| "naucs_at_10_std": 0.25898837879862296, | |
| "naucs_at_10_diff1": -0.0032105086289669817, | |
| "naucs_at_20_max": 0.1701364069679977, | |
| "naucs_at_20_std": 0.3491634058276955, | |
| "naucs_at_20_diff1": -0.10811392645442519, | |
| "naucs_at_50_max": 0.22116549189926218, | |
| "naucs_at_50_std": 0.40046893166130704, | |
| "naucs_at_50_diff1": -0.22625061791645093, | |
| "naucs_at_100_max": 0.23011945384723256, | |
| "naucs_at_100_std": 0.42936675975120925, | |
| "naucs_at_100_diff1": -0.2960643771988182 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { | |
| "ndcg_at_1": 0.54375, | |
| "ndcg_at_3": 0.52916, | |
| "ndcg_at_5": 0.54779, | |
| "ndcg_at_10": 0.58771, | |
| "ndcg_at_20": 0.61859, | |
| "ndcg_at_50": 0.64913, | |
| "ndcg_at_100": 0.66185, | |
| "map_at_1": 0.33558, | |
| "map_at_3": 0.4307, | |
| "map_at_5": 0.46539, | |
| "map_at_10": 0.49662, | |
| "map_at_20": 0.51175, | |
| "map_at_50": 0.52171, | |
| "map_at_100": 0.52497, | |
| "recall_at_1": 0.33558, | |
| "recall_at_3": 0.49142, | |
| "recall_at_5": 0.5656, | |
| "recall_at_10": 0.67964, | |
| "recall_at_20": 0.77075, | |
| "recall_at_50": 0.87085, | |
| "recall_at_100": 0.91455, | |
| "precision_at_1": 0.54375, | |
| "precision_at_3": 0.32292, | |
| "precision_at_5": 0.24625, | |
| "precision_at_10": 0.16063, | |
| "precision_at_20": 0.09688, | |
| "precision_at_50": 0.04762, | |
| "precision_at_100": 0.02675, | |
| "mrr_at_1": 0.54375, | |
| "mrr_at_3": 0.6177083333333331, | |
| "mrr_at_5": 0.6317708333333332, | |
| "mrr_at_10": 0.6458953373015872, | |
| "mrr_at_20": 0.6501168388392652, | |
| "mrr_at_50": 0.6516168070330217, | |
| "mrr_at_100": 0.6516168070330217, | |
| "naucs_at_1_max": 0.17394147030363272, | |
| "naucs_at_1_std": -0.36673902241284584, | |
| "naucs_at_1_diff1": 0.43317554550130327, | |
| "naucs_at_3_max": 0.10230735338450472, | |
| "naucs_at_3_std": -0.16796141270319767, | |
| "naucs_at_3_diff1": -0.0021937249263731763, | |
| "naucs_at_5_max": 0.06165269158937461, | |
| "naucs_at_5_std": -0.16213659251057302, | |
| "naucs_at_5_diff1": -0.09605836244564413, | |
| "naucs_at_10_max": 0.04062826287163538, | |
| "naucs_at_10_std": -0.10386324001578753, | |
| "naucs_at_10_diff1": -0.19533141678260185, | |
| "naucs_at_20_max": -0.023024941185161928, | |
| "naucs_at_20_std": -0.0808216300079185, | |
| "naucs_at_20_diff1": -0.2571645591830793, | |
| "naucs_at_50_max": -0.06194972750127154, | |
| "naucs_at_50_std": 0.023927949244181214, | |
| "naucs_at_50_diff1": -0.28301815335673003, | |
| "naucs_at_100_max": -0.10101655678389299, | |
| "naucs_at_100_std": 0.08336119059224785, | |
| "naucs_at_100_diff1": -0.2955701804119539 | |
| }, | |
| "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { | |
| "ndcg_at_1": 0.63793, | |
| "ndcg_at_3": 0.61396, | |
| "ndcg_at_5": 0.59648, | |
| "ndcg_at_10": 0.55957, | |
| "ndcg_at_20": 0.58035, | |
| "ndcg_at_50": 0.64249, | |
| "ndcg_at_100": 0.68124, | |
| "map_at_1": 0.07146, | |
| "map_at_3": 0.16727, | |
| "map_at_5": 0.22845, | |
| "map_at_10": 0.29412, | |
| "map_at_20": 0.35326, | |
| "map_at_50": 0.41154, | |
| "map_at_100": 0.43875, | |
| "recall_at_1": 0.07146, | |
| "recall_at_3": 0.2114, | |
| "recall_at_5": 0.31582, | |
| "recall_at_10": 0.44705, | |
| "recall_at_20": 0.59174, | |
| "recall_at_50": 0.77882, | |
| "recall_at_100": 0.90011, | |
| "precision_at_1": 0.63793, | |
| "precision_at_3": 0.58046, | |
| "precision_at_5": 0.53448, | |
| "precision_at_10": 0.4069, | |
| "precision_at_20": 0.31121, | |
| "precision_at_50": 0.19621, | |
| "precision_at_100": 0.1269, | |
| "mrr_at_1": 0.6379310344827587, | |
| "mrr_at_3": 0.735632183908046, | |
| "mrr_at_5": 0.7502873563218392, | |
| "mrr_at_10": 0.7584975369458129, | |
| "mrr_at_20": 0.7607245484400658, | |
| "mrr_at_50": 0.7607245484400658, | |
| "mrr_at_100": 0.7607245484400658, | |
| "naucs_at_1_max": 0.30951896933248446, | |
| "naucs_at_1_std": 0.0009385908345415833, | |
| "naucs_at_1_diff1": 0.12207984020363619, | |
| "naucs_at_3_max": 0.3174256751774014, | |
| "naucs_at_3_std": 0.07401442252929685, | |
| "naucs_at_3_diff1": 0.0813575535006513, | |
| "naucs_at_5_max": 0.3929619603728344, | |
| "naucs_at_5_std": 0.1689559171748497, | |
| "naucs_at_5_diff1": 0.04881016825842116, | |
| "naucs_at_10_max": 0.4375324160420264, | |
| "naucs_at_10_std": 0.2657788992938007, | |
| "naucs_at_10_diff1": -0.08210009736139143, | |
| "naucs_at_20_max": 0.435316184966006, | |
| "naucs_at_20_std": 0.3991338622440554, | |
| "naucs_at_20_diff1": -0.0429704940007693, | |
| "naucs_at_50_max": 0.3483632241805499, | |
| "naucs_at_50_std": 0.33986582587288133, | |
| "naucs_at_50_diff1": 0.0168280192203874, | |
| "naucs_at_100_max": 0.28796239161632026, | |
| "naucs_at_100_std": 0.27465381207019524, | |
| "naucs_at_100_diff1": 0.04462948923164142 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0": { | |
| "ndcg_at_1": 0.42105, | |
| "ndcg_at_3": 0.40524, | |
| "ndcg_at_5": 0.45864, | |
| "ndcg_at_10": 0.50339, | |
| "ndcg_at_20": 0.52015, | |
| "ndcg_at_50": 0.55124, | |
| "ndcg_at_100": 0.58182, | |
| "map_at_1": 0.20117, | |
| "map_at_3": 0.29694, | |
| "map_at_5": 0.34743, | |
| "map_at_10": 0.38591, | |
| "map_at_20": 0.39764, | |
| "map_at_50": 0.40953, | |
| "map_at_100": 0.41648, | |
| "recall_at_1": 0.20117, | |
| "recall_at_3": 0.35925, | |
| "recall_at_5": 0.49808, | |
| "recall_at_10": 0.6391, | |
| "recall_at_20": 0.69492, | |
| "recall_at_50": 0.79371, | |
| "recall_at_100": 0.91345, | |
| "precision_at_1": 0.42105, | |
| "precision_at_3": 0.26901, | |
| "precision_at_5": 0.24561, | |
| "precision_at_10": 0.16842, | |
| "precision_at_20": 0.09912, | |
| "precision_at_50": 0.05298, | |
| "precision_at_100": 0.03316, | |
| "mrr_at_1": 0.42105263157894735, | |
| "mrr_at_3": 0.5058479532163742, | |
| "mrr_at_5": 0.5330409356725145, | |
| "mrr_at_10": 0.5501461988304093, | |
| "mrr_at_20": 0.5501461988304093, | |
| "mrr_at_50": 0.5521026864735273, | |
| "mrr_at_100": 0.5529842660756151, | |
| "naucs_at_1_max": 0.2681727937415602, | |
| "naucs_at_1_std": 0.26125468443050986, | |
| "naucs_at_1_diff1": 0.1925487726366543, | |
| "naucs_at_3_max": 0.15154876755279928, | |
| "naucs_at_3_std": 0.187284520490265, | |
| "naucs_at_3_diff1": 0.13314931121987555, | |
| "naucs_at_5_max": 0.06746755549674449, | |
| "naucs_at_5_std": 0.17211166986401807, | |
| "naucs_at_5_diff1": 0.09922461829032554, | |
| "naucs_at_10_max": 0.1173297532733523, | |
| "naucs_at_10_std": 0.2334653355769548, | |
| "naucs_at_10_diff1": 0.13391976394738275, | |
| "naucs_at_20_max": -0.004406787213674995, | |
| "naucs_at_20_std": 0.2228534925354155, | |
| "naucs_at_20_diff1": 0.0419546501642916, | |
| "naucs_at_50_max": -0.09092484597679043, | |
| "naucs_at_50_std": 0.2015080134456558, | |
| "naucs_at_50_diff1": -0.06928136029689304, | |
| "naucs_at_100_max": -0.14425251784397497, | |
| "naucs_at_100_std": 0.2117646090542283, | |
| "naucs_at_100_diff1": -0.12598719754364754 | |
| }, | |
| "vidore/synthetic_axa_filtered_v1.0": { | |
| "ndcg_at_1": 0.38889, | |
| "ndcg_at_3": 0.5418, | |
| "ndcg_at_5": 0.50473, | |
| "ndcg_at_10": 0.53634, | |
| "ndcg_at_20": 0.56481, | |
| "ndcg_at_50": 0.60088, | |
| "ndcg_at_100": 0.62788, | |
| "map_at_1": 0.19085, | |
| "map_at_3": 0.33844, | |
| "map_at_5": 0.34887, | |
| "map_at_10": 0.3905, | |
| "map_at_20": 0.41424, | |
| "map_at_50": 0.42936, | |
| "map_at_100": 0.43807, | |
| "recall_at_1": 0.19085, | |
| "recall_at_3": 0.52023, | |
| "recall_at_5": 0.53824, | |
| "recall_at_10": 0.68819, | |
| "recall_at_20": 0.76306, | |
| "recall_at_50": 0.85789, | |
| "recall_at_100": 0.93331, | |
| "precision_at_1": 0.38889, | |
| "precision_at_3": 0.38889, | |
| "precision_at_5": 0.26667, | |
| "precision_at_10": 0.2, | |
| "precision_at_20": 0.13333, | |
| "precision_at_50": 0.06889, | |
| "precision_at_100": 0.04222, | |
| "mrr_at_1": 0.3888888888888889, | |
| "mrr_at_3": 0.5740740740740741, | |
| "mrr_at_5": 0.587962962962963, | |
| "mrr_at_10": 0.6033950617283952, | |
| "mrr_at_20": 0.6033950617283952, | |
| "mrr_at_50": 0.6033950617283952, | |
| "mrr_at_100": 0.6033950617283952, | |
| "naucs_at_1_max": 0.14105650005960155, | |
| "naucs_at_1_std": -0.01993661989253736, | |
| "naucs_at_1_diff1": 0.43542988491270046, | |
| "naucs_at_3_max": -0.22826796699446616, | |
| "naucs_at_3_std": -0.004572665724137645, | |
| "naucs_at_3_diff1": 0.07496811347896108, | |
| "naucs_at_5_max": -0.3852842673944422, | |
| "naucs_at_5_std": -0.15791921982494983, | |
| "naucs_at_5_diff1": 0.2052446052071864, | |
| "naucs_at_10_max": -0.3499853580695824, | |
| "naucs_at_10_std": -0.07324394708860461, | |
| "naucs_at_10_diff1": 0.06796721460372077, | |
| "naucs_at_20_max": -0.360118573152839, | |
| "naucs_at_20_std": -0.15283583044728602, | |
| "naucs_at_20_diff1": 0.26460065509124203, | |
| "naucs_at_50_max": -0.38416433266363276, | |
| "naucs_at_50_std": -0.1571206204390072, | |
| "naucs_at_50_diff1": 0.4313242188093156, | |
| "naucs_at_100_max": -0.4149515845406916, | |
| "naucs_at_100_std": -0.19401302943385454, | |
| "naucs_at_100_diff1": 0.49450789205035006 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.39912, | |
| "ndcg_at_3": 0.43071, | |
| "ndcg_at_5": 0.46444, | |
| "ndcg_at_10": 0.4968, | |
| "ndcg_at_20": 0.52854, | |
| "ndcg_at_50": 0.56102, | |
| "ndcg_at_100": 0.58233, | |
| "map_at_1": 0.17982, | |
| "map_at_3": 0.30157, | |
| "map_at_5": 0.34561, | |
| "map_at_10": 0.38009, | |
| "map_at_20": 0.39814, | |
| "map_at_50": 0.41046, | |
| "map_at_100": 0.41606, | |
| "recall_at_1": 0.17982, | |
| "recall_at_3": 0.4008, | |
| "recall_at_5": 0.50897, | |
| "recall_at_10": 0.62213, | |
| "recall_at_20": 0.72976, | |
| "recall_at_50": 0.83655, | |
| "recall_at_100": 0.915, | |
| "precision_at_1": 0.39912, | |
| "precision_at_3": 0.30263, | |
| "precision_at_5": 0.25175, | |
| "precision_at_10": 0.16886, | |
| "precision_at_20": 0.10636, | |
| "precision_at_50": 0.05632, | |
| "precision_at_100": 0.03329, | |
| "mrr_at_1": 0.3991228070175439, | |
| "mrr_at_3": 0.5160818713450291, | |
| "mrr_at_5": 0.5342836257309941, | |
| "mrr_at_10": 0.5459934558618768, | |
| "mrr_at_20": 0.5499615268119828, | |
| "mrr_at_50": 0.5520170407199186, | |
| "mrr_at_100": 0.5523559958857708, | |
| "naucs_at_1_max": 0.15755765375477634, | |
| "naucs_at_1_std": 0.1937615823895248, | |
| "naucs_at_1_diff1": 0.20176410519897442, | |
| "naucs_at_3_max": 0.21637882247590812, | |
| "naucs_at_3_std": 0.21919003966069206, | |
| "naucs_at_3_diff1": 0.09426673538244577, | |
| "naucs_at_5_max": 0.17022232642187712, | |
| "naucs_at_5_std": 0.20399998951672477, | |
| "naucs_at_5_diff1": 0.07433346087483172, | |
| "naucs_at_10_max": 0.17445157988680682, | |
| "naucs_at_10_std": 0.19671403249308644, | |
| "naucs_at_10_diff1": 0.042552504976369365, | |
| "naucs_at_20_max": 0.0564754707494715, | |
| "naucs_at_20_std": 0.14487480859508034, | |
| "naucs_at_20_diff1": -0.050996649070789594, | |
| "naucs_at_50_max": -0.032342844885820655, | |
| "naucs_at_50_std": 0.0919650425307495, | |
| "naucs_at_50_diff1": -0.1049317890928347, | |
| "naucs_at_100_max": -0.0813860336462654, | |
| "naucs_at_100_std": 0.07405564056158524, | |
| "naucs_at_100_diff1": -0.15314098265443818 | |
| }, | |
| "vidore/synthetic_axa_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.40278, | |
| "ndcg_at_3": 0.45413, | |
| "ndcg_at_5": 0.45175, | |
| "ndcg_at_10": 0.49692, | |
| "ndcg_at_20": 0.53744, | |
| "ndcg_at_50": 0.57505, | |
| "ndcg_at_100": 0.60205, | |
| "map_at_1": 0.20688, | |
| "map_at_3": 0.30245, | |
| "map_at_5": 0.32956, | |
| "map_at_10": 0.37191, | |
| "map_at_20": 0.39679, | |
| "map_at_50": 0.41224, | |
| "map_at_100": 0.42049, | |
| "recall_at_1": 0.20688, | |
| "recall_at_3": 0.39577, | |
| "recall_at_5": 0.462, | |
| "recall_at_10": 0.6319, | |
| "recall_at_20": 0.73905, | |
| "recall_at_50": 0.83135, | |
| "recall_at_100": 0.90763, | |
| "precision_at_1": 0.40278, | |
| "precision_at_3": 0.30556, | |
| "precision_at_5": 0.23889, | |
| "precision_at_10": 0.17917, | |
| "precision_at_20": 0.125, | |
| "precision_at_50": 0.06639, | |
| "precision_at_100": 0.04097, | |
| "mrr_at_1": 0.4027777777777778, | |
| "mrr_at_3": 0.5300925925925926, | |
| "mrr_at_5": 0.5564814814814815, | |
| "mrr_at_10": 0.5736221340388007, | |
| "mrr_at_20": 0.5763117283950616, | |
| "mrr_at_50": 0.5763117283950616, | |
| "mrr_at_100": 0.5763117283950616, | |
| "naucs_at_1_max": 0.20656968495031644, | |
| "naucs_at_1_std": 0.0817297457698203, | |
| "naucs_at_1_diff1": 0.2238136061191017, | |
| "naucs_at_3_max": 0.25163264129737517, | |
| "naucs_at_3_std": 0.17030727986741342, | |
| "naucs_at_3_diff1": 0.03578498764457108, | |
| "naucs_at_5_max": 0.08903441500332349, | |
| "naucs_at_5_std": 0.04568596558397242, | |
| "naucs_at_5_diff1": -0.009941530139102166, | |
| "naucs_at_10_max": -0.020546119491255295, | |
| "naucs_at_10_std": 0.008887544287522966, | |
| "naucs_at_10_diff1": -0.09432640328513163, | |
| "naucs_at_20_max": -0.1319344233352006, | |
| "naucs_at_20_std": -0.044221131984674825, | |
| "naucs_at_20_diff1": -0.06538820564150717, | |
| "naucs_at_50_max": -0.18743175691838732, | |
| "naucs_at_50_std": -0.0972663161231581, | |
| "naucs_at_50_diff1": -0.017105606627328275, | |
| "naucs_at_100_max": -0.2325355500441028, | |
| "naucs_at_100_std": -0.15406419774878136, | |
| "naucs_at_100_diff1": -0.018290038816855033 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { | |
| "ndcg_at_1": 0.43438, | |
| "ndcg_at_3": 0.45593, | |
| "ndcg_at_5": 0.47695, | |
| "ndcg_at_10": 0.5166, | |
| "ndcg_at_20": 0.54502, | |
| "ndcg_at_50": 0.57616, | |
| "ndcg_at_100": 0.59352, | |
| "map_at_1": 0.27154, | |
| "map_at_3": 0.36745, | |
| "map_at_5": 0.39714, | |
| "map_at_10": 0.42629, | |
| "map_at_20": 0.43946, | |
| "map_at_50": 0.44896, | |
| "map_at_100": 0.45239, | |
| "recall_at_1": 0.27154, | |
| "recall_at_3": 0.44101, | |
| "recall_at_5": 0.51211, | |
| "recall_at_10": 0.62263, | |
| "recall_at_20": 0.70666, | |
| "recall_at_50": 0.81063, | |
| "recall_at_100": 0.87902, | |
| "precision_at_1": 0.43438, | |
| "precision_at_3": 0.27917, | |
| "precision_at_5": 0.21531, | |
| "precision_at_10": 0.14359, | |
| "precision_at_20": 0.08672, | |
| "precision_at_50": 0.04341, | |
| "precision_at_100": 0.02502, | |
| "mrr_at_1": 0.434375, | |
| "mrr_at_3": 0.5317708333333331, | |
| "mrr_at_5": 0.5496614583333328, | |
| "mrr_at_10": 0.5630952380952378, | |
| "mrr_at_20": 0.5669521784558423, | |
| "mrr_at_50": 0.5688358670627492, | |
| "mrr_at_100": 0.5692193060123717, | |
| "naucs_at_1_max": 0.2666912240570166, | |
| "naucs_at_1_std": 0.05389058257999144, | |
| "naucs_at_1_diff1": 0.4321239157776483, | |
| "naucs_at_3_max": 0.17727410640755303, | |
| "naucs_at_3_std": 0.01577475274662206, | |
| "naucs_at_3_diff1": 0.09089286579349252, | |
| "naucs_at_5_max": 0.13070625524250945, | |
| "naucs_at_5_std": 0.01198523787261333, | |
| "naucs_at_5_diff1": -0.013247813819131482, | |
| "naucs_at_10_max": 0.08901372928565265, | |
| "naucs_at_10_std": 0.0242424061104149, | |
| "naucs_at_10_diff1": -0.0824662153065525, | |
| "naucs_at_20_max": 0.046509940234363965, | |
| "naucs_at_20_std": 0.023421019939509093, | |
| "naucs_at_20_diff1": -0.13765222771211638, | |
| "naucs_at_50_max": -0.016368222256068723, | |
| "naucs_at_50_std": 0.017686339762030524, | |
| "naucs_at_50_diff1": -0.18185950361763814, | |
| "naucs_at_100_max": -0.05809760099241634, | |
| "naucs_at_100_std": -0.0007126993337199372, | |
| "naucs_at_100_diff1": -0.20245463094499655 | |
| }, | |
| "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.52586, | |
| "ndcg_at_3": 0.49942, | |
| "ndcg_at_5": 0.48681, | |
| "ndcg_at_10": 0.46438, | |
| "ndcg_at_20": 0.49413, | |
| "ndcg_at_50": 0.56018, | |
| "ndcg_at_100": 0.60792, | |
| "map_at_1": 0.05323, | |
| "map_at_3": 0.12858, | |
| "map_at_5": 0.17351, | |
| "map_at_10": 0.22551, | |
| "map_at_20": 0.27667, | |
| "map_at_50": 0.3298, | |
| "map_at_100": 0.35718, | |
| "recall_at_1": 0.05323, | |
| "recall_at_3": 0.16936, | |
| "recall_at_5": 0.25112, | |
| "recall_at_10": 0.37154, | |
| "recall_at_20": 0.52494, | |
| "recall_at_50": 0.7231, | |
| "recall_at_100": 0.86824, | |
| "precision_at_1": 0.52586, | |
| "precision_at_3": 0.4727, | |
| "precision_at_5": 0.43966, | |
| "precision_at_10": 0.35, | |
| "precision_at_20": 0.27651, | |
| "precision_at_50": 0.17888, | |
| "precision_at_100": 0.12013, | |
| "mrr_at_1": 0.5258620689655172, | |
| "mrr_at_3": 0.6163793103448275, | |
| "mrr_at_5": 0.6372844827586207, | |
| "mrr_at_10": 0.6511271893814995, | |
| "mrr_at_20": 0.6548831549338646, | |
| "mrr_at_50": 0.6554856928251602, | |
| "mrr_at_100": 0.6556279827912883, | |
| "naucs_at_1_max": 0.1755971141206357, | |
| "naucs_at_1_std": 0.18264029674960228, | |
| "naucs_at_1_diff1": 0.06925322234490841, | |
| "naucs_at_3_max": 0.2812764433128787, | |
| "naucs_at_3_std": 0.2690356636124753, | |
| "naucs_at_3_diff1": 0.016089000880329198, | |
| "naucs_at_5_max": 0.283989396373602, | |
| "naucs_at_5_std": 0.29562037646588923, | |
| "naucs_at_5_diff1": 0.009509118422791094, | |
| "naucs_at_10_max": 0.23668581693373705, | |
| "naucs_at_10_std": 0.2844259803726197, | |
| "naucs_at_10_diff1": -0.07126374015391594, | |
| "naucs_at_20_max": 0.2076351063428368, | |
| "naucs_at_20_std": 0.2991176793010035, | |
| "naucs_at_20_diff1": -0.08337920885811372, | |
| "naucs_at_50_max": 0.1238437265668122, | |
| "naucs_at_50_std": 0.2430342456930391, | |
| "naucs_at_50_diff1": -0.05232657907650674, | |
| "naucs_at_100_max": 0.05825862012096387, | |
| "naucs_at_100_std": 0.19337982478846943, | |
| "naucs_at_100_diff1": -0.020634764162514304 | |
| }, | |
| "vidore/arxivqa_test_subsampled": { | |
| "ndcg_at_1": 0.736, | |
| "ndcg_at_3": 0.79238, | |
| "ndcg_at_5": 0.80581, | |
| "ndcg_at_10": 0.82083, | |
| "ndcg_at_20": 0.82784, | |
| "ndcg_at_50": 0.83516, | |
| "ndcg_at_100": 0.83776, | |
| "map_at_1": 0.736, | |
| "map_at_3": 0.77867, | |
| "map_at_5": 0.78627, | |
| "map_at_10": 0.79255, | |
| "map_at_20": 0.79443, | |
| "map_at_50": 0.79568, | |
| "map_at_100": 0.79591, | |
| "recall_at_1": 0.736, | |
| "recall_at_3": 0.832, | |
| "recall_at_5": 0.864, | |
| "recall_at_10": 0.91, | |
| "recall_at_20": 0.938, | |
| "recall_at_50": 0.974, | |
| "recall_at_100": 0.99, | |
| "precision_at_1": 0.736, | |
| "precision_at_3": 0.27733, | |
| "precision_at_5": 0.1728, | |
| "precision_at_10": 0.091, | |
| "precision_at_20": 0.0469, | |
| "precision_at_50": 0.01948, | |
| "precision_at_100": 0.0099, | |
| "mrr_at_1": 0.736, | |
| "mrr_at_3": 0.7786666666666667, | |
| "mrr_at_5": 0.7862666666666667, | |
| "mrr_at_10": 0.7925476190476194, | |
| "mrr_at_20": 0.7944328455428302, | |
| "mrr_at_50": 0.7956750456398185, | |
| "mrr_at_100": 0.7959050774896889, | |
| "naucs_at_1_max": 0.6486039470733347, | |
| "naucs_at_1_std": -0.11010636593726965, | |
| "naucs_at_1_diff1": 0.8885840835549293, | |
| "naucs_at_3_max": 0.6259037134406845, | |
| "naucs_at_3_std": -0.18474484765973373, | |
| "naucs_at_3_diff1": 0.8376660720153996, | |
| "naucs_at_5_max": 0.6215153325368371, | |
| "naucs_at_5_std": -0.17638817773226517, | |
| "naucs_at_5_diff1": 0.8335253456221203, | |
| "naucs_at_10_max": 0.5732648615001562, | |
| "naucs_at_10_std": -0.2490196078431399, | |
| "naucs_at_10_diff1": 0.787851436871045, | |
| "naucs_at_20_max": 0.5872112285774529, | |
| "naucs_at_20_std": -0.18579259660853667, | |
| "naucs_at_20_diff1": 0.8300502996897676, | |
| "naucs_at_50_max": 0.8495654672125184, | |
| "naucs_at_50_std": 0.27152912447029887, | |
| "naucs_at_50_diff1": 0.8371399841988043, | |
| "naucs_at_100_max": 0.9183006535947666, | |
| "naucs_at_100_std": 0.49262371615312944, | |
| "naucs_at_100_diff1": 0.8921568627450854 | |
| }, | |
| "vidore/docvqa_test_subsampled": { | |
| "ndcg_at_1": 0.34368, | |
| "ndcg_at_3": 0.40479, | |
| "ndcg_at_5": 0.42673, | |
| "ndcg_at_10": 0.45035, | |
| "ndcg_at_20": 0.46746, | |
| "ndcg_at_50": 0.48358, | |
| "ndcg_at_100": 0.49656, | |
| "map_at_1": 0.34368, | |
| "map_at_3": 0.38987, | |
| "map_at_5": 0.40207, | |
| "map_at_10": 0.41179, | |
| "map_at_20": 0.41664, | |
| "map_at_50": 0.41935, | |
| "map_at_100": 0.42051, | |
| "recall_at_1": 0.34368, | |
| "recall_at_3": 0.44789, | |
| "recall_at_5": 0.50111, | |
| "recall_at_10": 0.57428, | |
| "recall_at_20": 0.6408, | |
| "recall_at_50": 0.72062, | |
| "recall_at_100": 0.80044, | |
| "precision_at_1": 0.34368, | |
| "precision_at_3": 0.1493, | |
| "precision_at_5": 0.10022, | |
| "precision_at_10": 0.05743, | |
| "precision_at_20": 0.03204, | |
| "precision_at_50": 0.01441, | |
| "precision_at_100": 0.008, | |
| "mrr_at_1": 0.3436807095343681, | |
| "mrr_at_3": 0.3898743532889875, | |
| "mrr_at_5": 0.4020694752402069, | |
| "mrr_at_10": 0.4117877380072501, | |
| "mrr_at_20": 0.4166391095440233, | |
| "mrr_at_50": 0.4194072008524503, | |
| "mrr_at_100": 0.42066245247914874, | |
| "naucs_at_1_max": 0.6937621431789922, | |
| "naucs_at_1_std": 0.3966932403411503, | |
| "naucs_at_1_diff1": 0.7145229983797862, | |
| "naucs_at_3_max": 0.7320406126458378, | |
| "naucs_at_3_std": 0.45563802569337475, | |
| "naucs_at_3_diff1": 0.6143081963671839, | |
| "naucs_at_5_max": 0.6880199080240672, | |
| "naucs_at_5_std": 0.44498830503027725, | |
| "naucs_at_5_diff1": 0.5636895248104472, | |
| "naucs_at_10_max": 0.7395456562744627, | |
| "naucs_at_10_std": 0.5421201928794931, | |
| "naucs_at_10_diff1": 0.5433914071660855, | |
| "naucs_at_20_max": 0.7099358788663228, | |
| "naucs_at_20_std": 0.505428632850115, | |
| "naucs_at_20_diff1": 0.5026852667645294, | |
| "naucs_at_50_max": 0.7044396744042806, | |
| "naucs_at_50_std": 0.5292950459939872, | |
| "naucs_at_50_diff1": 0.48465037492486385, | |
| "naucs_at_100_max": 0.6982871591556156, | |
| "naucs_at_100_std": 0.5542470657836935, | |
| "naucs_at_100_diff1": 0.4520088087347075 | |
| }, | |
| "vidore/infovqa_test_subsampled": { | |
| "ndcg_at_1": 0.76923, | |
| "ndcg_at_3": 0.83858, | |
| "ndcg_at_5": 0.85026, | |
| "ndcg_at_10": 0.86321, | |
| "ndcg_at_20": 0.86616, | |
| "ndcg_at_50": 0.86899, | |
| "ndcg_at_100": 0.86929, | |
| "map_at_1": 0.76923, | |
| "map_at_3": 0.82254, | |
| "map_at_5": 0.82901, | |
| "map_at_10": 0.83467, | |
| "map_at_20": 0.83541, | |
| "map_at_50": 0.83587, | |
| "map_at_100": 0.8359, | |
| "recall_at_1": 0.76923, | |
| "recall_at_3": 0.88462, | |
| "recall_at_5": 0.91296, | |
| "recall_at_10": 0.95142, | |
| "recall_at_20": 0.96356, | |
| "recall_at_50": 0.97773, | |
| "recall_at_100": 0.97976, | |
| "precision_at_1": 0.76923, | |
| "precision_at_3": 0.29487, | |
| "precision_at_5": 0.18259, | |
| "precision_at_10": 0.09514, | |
| "precision_at_20": 0.04818, | |
| "precision_at_50": 0.01955, | |
| "precision_at_100": 0.0098, | |
| "mrr_at_1": 0.7692307692307693, | |
| "mrr_at_3": 0.8225371120107962, | |
| "mrr_at_5": 0.8290148448043184, | |
| "mrr_at_10": 0.8346667951931113, | |
| "mrr_at_20": 0.8354146169277593, | |
| "mrr_at_50": 0.8358748221328831, | |
| "mrr_at_100": 0.8359177231300928, | |
| "naucs_at_1_max": 0.5127634005653434, | |
| "naucs_at_1_std": -0.03567831485698759, | |
| "naucs_at_1_diff1": 0.8320638205788135, | |
| "naucs_at_3_max": 0.5021250200120738, | |
| "naucs_at_3_std": -0.07399077187048693, | |
| "naucs_at_3_diff1": 0.7752634768446268, | |
| "naucs_at_5_max": 0.5335848645863028, | |
| "naucs_at_5_std": -0.02593966089833383, | |
| "naucs_at_5_diff1": 0.7661635495872359, | |
| "naucs_at_10_max": 0.9000642537485549, | |
| "naucs_at_10_std": 0.4822680430176945, | |
| "naucs_at_10_diff1": 0.7890895385688544, | |
| "naucs_at_20_max": 0.8894376823215577, | |
| "naucs_at_20_std": 0.4867325787808216, | |
| "naucs_at_20_diff1": 0.7414713954152902, | |
| "naucs_at_50_max": 0.9762542810791424, | |
| "naucs_at_50_std": 0.9525085621582665, | |
| "naucs_at_50_diff1": 0.733736269334063, | |
| "naucs_at_100_max": 0.9738797091870466, | |
| "naucs_at_100_std": 0.960819563780575, | |
| "naucs_at_100_diff1": 0.7348833700432545 | |
| }, | |
| "vidore/tabfquad_test_subsampled": { | |
| "ndcg_at_1": 0.69286, | |
| "ndcg_at_3": 0.75259, | |
| "ndcg_at_5": 0.76889, | |
| "ndcg_at_10": 0.79021, | |
| "ndcg_at_20": 0.80681, | |
| "ndcg_at_50": 0.81245, | |
| "ndcg_at_100": 0.81307, | |
| "map_at_1": 0.69286, | |
| "map_at_3": 0.7375, | |
| "map_at_5": 0.74661, | |
| "map_at_10": 0.75505, | |
| "map_at_20": 0.75979, | |
| "map_at_50": 0.76068, | |
| "map_at_100": 0.76074, | |
| "recall_at_1": 0.69286, | |
| "recall_at_3": 0.79643, | |
| "recall_at_5": 0.83571, | |
| "recall_at_10": 0.90357, | |
| "recall_at_20": 0.96786, | |
| "recall_at_50": 0.99643, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.69286, | |
| "precision_at_3": 0.26548, | |
| "precision_at_5": 0.16714, | |
| "precision_at_10": 0.09036, | |
| "precision_at_20": 0.04839, | |
| "precision_at_50": 0.01993, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.6928571428571428, | |
| "mrr_at_3": 0.7375000000000002, | |
| "mrr_at_5": 0.746607142857143, | |
| "mrr_at_10": 0.7550481859410433, | |
| "mrr_at_20": 0.7597886651746948, | |
| "mrr_at_50": 0.7606781401931197, | |
| "mrr_at_100": 0.7607430752580548, | |
| "naucs_at_1_max": 0.6018301415698155, | |
| "naucs_at_1_std": 0.32571196816263426, | |
| "naucs_at_1_diff1": 0.7888873398309274, | |
| "naucs_at_3_max": 0.6444276300161007, | |
| "naucs_at_3_std": 0.3241186355677946, | |
| "naucs_at_3_diff1": 0.7616114869917906, | |
| "naucs_at_5_max": 0.6430603688016092, | |
| "naucs_at_5_std": 0.32898001968082796, | |
| "naucs_at_5_diff1": 0.725377572412612, | |
| "naucs_at_10_max": 0.5787598990213383, | |
| "naucs_at_10_std": 0.39082546598886503, | |
| "naucs_at_10_diff1": 0.6965107030466515, | |
| "naucs_at_20_max": 0.5347546425977789, | |
| "naucs_at_20_std": 0.10488639900404371, | |
| "naucs_at_20_diff1": 0.5825293080195045, | |
| "naucs_at_50_max": 0.8692810457515607, | |
| "naucs_at_50_std": 0.5541549953314449, | |
| "naucs_at_50_diff1": 0.8692810457515607, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 1.0, | |
| "naucs_at_100_diff1": 1.0 | |
| }, | |
| "vidore/tatdqa_test": { | |
| "ndcg_at_1": 0.35115, | |
| "ndcg_at_3": 0.45665, | |
| "ndcg_at_5": 0.49299, | |
| "ndcg_at_10": 0.53515, | |
| "ndcg_at_20": 0.55722, | |
| "ndcg_at_50": 0.57662, | |
| "ndcg_at_100": 0.58344, | |
| "map_at_1": 0.35115, | |
| "map_at_3": 0.43074, | |
| "map_at_5": 0.45094, | |
| "map_at_10": 0.46853, | |
| "map_at_20": 0.47464, | |
| "map_at_50": 0.47794, | |
| "map_at_100": 0.47855, | |
| "recall_at_1": 0.35115, | |
| "recall_at_3": 0.53159, | |
| "recall_at_5": 0.61968, | |
| "recall_at_10": 0.74909, | |
| "recall_at_20": 0.83597, | |
| "recall_at_50": 0.93135, | |
| "recall_at_100": 0.97327, | |
| "precision_at_1": 0.35115, | |
| "precision_at_3": 0.1772, | |
| "precision_at_5": 0.12394, | |
| "precision_at_10": 0.07491, | |
| "precision_at_20": 0.0418, | |
| "precision_at_50": 0.01863, | |
| "precision_at_100": 0.00973, | |
| "mrr_at_1": 0.35297691373025514, | |
| "mrr_at_3": 0.431652490886999, | |
| "mrr_at_5": 0.45188335358444764, | |
| "mrr_at_10": 0.4694830179945614, | |
| "mrr_at_20": 0.47559566338222015, | |
| "mrr_at_50": 0.47890243738258254, | |
| "mrr_at_100": 0.47950989875566846, | |
| "naucs_at_1_max": 0.16045300557832828, | |
| "naucs_at_1_std": -0.18712590459385856, | |
| "naucs_at_1_diff1": 0.5440702710166525, | |
| "naucs_at_3_max": 0.20647987331228912, | |
| "naucs_at_3_std": -0.12049997692079921, | |
| "naucs_at_3_diff1": 0.4450522896579701, | |
| "naucs_at_5_max": 0.2514186053554693, | |
| "naucs_at_5_std": -0.07250138069379193, | |
| "naucs_at_5_diff1": 0.43314416061424443, | |
| "naucs_at_10_max": 0.31938596426902055, | |
| "naucs_at_10_std": -0.03160768541250311, | |
| "naucs_at_10_diff1": 0.4063667061259836, | |
| "naucs_at_20_max": 0.37555947710006393, | |
| "naucs_at_20_std": 0.07079816177576714, | |
| "naucs_at_20_diff1": 0.4080320503888041, | |
| "naucs_at_50_max": 0.37314614111817646, | |
| "naucs_at_50_std": 0.15009417772189595, | |
| "naucs_at_50_diff1": 0.3940999242861563, | |
| "naucs_at_100_max": 0.5170529950663407, | |
| "naucs_at_100_std": 0.41066786010599293, | |
| "naucs_at_100_diff1": 0.41541021771983067 | |
| }, | |
| "vidore/shiftproject_test": { | |
| "ndcg_at_1": 0.44, | |
| "ndcg_at_3": 0.59095, | |
| "ndcg_at_5": 0.61204, | |
| "ndcg_at_10": 0.6477, | |
| "ndcg_at_20": 0.6722, | |
| "ndcg_at_50": 0.67625, | |
| "ndcg_at_100": 0.678, | |
| "map_at_1": 0.44, | |
| "map_at_3": 0.55333, | |
| "map_at_5": 0.56533, | |
| "map_at_10": 0.58014, | |
| "map_at_20": 0.58646, | |
| "map_at_50": 0.58714, | |
| "map_at_100": 0.58733, | |
| "recall_at_1": 0.44, | |
| "recall_at_3": 0.7, | |
| "recall_at_5": 0.75, | |
| "recall_at_10": 0.86, | |
| "recall_at_20": 0.96, | |
| "recall_at_50": 0.98, | |
| "recall_at_100": 0.99, | |
| "precision_at_1": 0.44, | |
| "precision_at_3": 0.23333, | |
| "precision_at_5": 0.15, | |
| "precision_at_10": 0.086, | |
| "precision_at_20": 0.048, | |
| "precision_at_50": 0.0196, | |
| "precision_at_100": 0.0099, | |
| "mrr_at_1": 0.44, | |
| "mrr_at_3": 0.5533333333333333, | |
| "mrr_at_5": 0.5653333333333334, | |
| "mrr_at_10": 0.580138888888889, | |
| "mrr_at_20": 0.5864581566094724, | |
| "mrr_at_50": 0.5871363175290126, | |
| "mrr_at_100": 0.5873286252213203, | |
| "naucs_at_1_max": -0.004671921490838898, | |
| "naucs_at_1_std": -0.12310121626559477, | |
| "naucs_at_1_diff1": 0.3854335229942058, | |
| "naucs_at_3_max": 0.06802377949918934, | |
| "naucs_at_3_std": -0.11448387677895848, | |
| "naucs_at_3_diff1": 0.31131327688704724, | |
| "naucs_at_5_max": -0.021580324437466547, | |
| "naucs_at_5_std": -0.21312401883830379, | |
| "naucs_at_5_diff1": 0.41149136577707984, | |
| "naucs_at_10_max": -0.1736871276153512, | |
| "naucs_at_10_std": -0.35710821671054294, | |
| "naucs_at_10_diff1": 0.2720659553831241, | |
| "naucs_at_20_max": 0.22934173669468105, | |
| "naucs_at_20_std": -0.1219654528478032, | |
| "naucs_at_20_diff1": 0.7864145658263278, | |
| "naucs_at_50_max": 0.6790382819794637, | |
| "naucs_at_50_std": 0.613678804855281, | |
| "naucs_at_50_diff1": 0.795751633986937, | |
| "naucs_at_100_max": 0.35807656395892007, | |
| "naucs_at_100_std": 0.35807656395892007, | |
| "naucs_at_100_diff1": 0.8692810457516374 | |
| }, | |
| "vidore/syntheticDocQA_artificial_intelligence_test": { | |
| "ndcg_at_1": 0.88, | |
| "ndcg_at_3": 0.93547, | |
| "ndcg_at_5": 0.94365, | |
| "ndcg_at_10": 0.94365, | |
| "ndcg_at_20": 0.94644, | |
| "ndcg_at_50": 0.94644, | |
| "ndcg_at_100": 0.94644, | |
| "map_at_1": 0.88, | |
| "map_at_3": 0.92333, | |
| "map_at_5": 0.92783, | |
| "map_at_10": 0.92783, | |
| "map_at_20": 0.92874, | |
| "map_at_50": 0.92874, | |
| "map_at_100": 0.92874, | |
| "recall_at_1": 0.88, | |
| "recall_at_3": 0.97, | |
| "recall_at_5": 0.99, | |
| "recall_at_10": 0.99, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.88, | |
| "precision_at_3": 0.32333, | |
| "precision_at_5": 0.198, | |
| "precision_at_10": 0.099, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.88, | |
| "mrr_at_3": 0.9233333333333335, | |
| "mrr_at_5": 0.9278333333333335, | |
| "mrr_at_10": 0.9278333333333335, | |
| "mrr_at_20": 0.9287424242424244, | |
| "mrr_at_50": 0.9287424242424244, | |
| "mrr_at_100": 0.9287424242424244, | |
| "naucs_at_1_max": 0.33646197900095476, | |
| "naucs_at_1_std": -0.10101813553929312, | |
| "naucs_at_1_diff1": 0.8668071905822462, | |
| "naucs_at_3_max": 0.8513849984438244, | |
| "naucs_at_3_std": 0.7424525365701778, | |
| "naucs_at_3_diff1": 0.664021164021167, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 0.8692810457516413, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 0.8692810457516413, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_energy_test": { | |
| "ndcg_at_1": 0.86, | |
| "ndcg_at_3": 0.92047, | |
| "ndcg_at_5": 0.92047, | |
| "ndcg_at_10": 0.92381, | |
| "ndcg_at_20": 0.92381, | |
| "ndcg_at_50": 0.92587, | |
| "ndcg_at_100": 0.92927, | |
| "map_at_1": 0.86, | |
| "map_at_3": 0.90667, | |
| "map_at_5": 0.90667, | |
| "map_at_10": 0.9081, | |
| "map_at_20": 0.9081, | |
| "map_at_50": 0.90845, | |
| "map_at_100": 0.9088, | |
| "recall_at_1": 0.86, | |
| "recall_at_3": 0.96, | |
| "recall_at_5": 0.96, | |
| "recall_at_10": 0.97, | |
| "recall_at_20": 0.97, | |
| "recall_at_50": 0.98, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.86, | |
| "precision_at_3": 0.32, | |
| "precision_at_5": 0.192, | |
| "precision_at_10": 0.097, | |
| "precision_at_20": 0.0485, | |
| "precision_at_50": 0.0196, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.86, | |
| "mrr_at_3": 0.9066666666666667, | |
| "mrr_at_5": 0.9066666666666667, | |
| "mrr_at_10": 0.9080952380952381, | |
| "mrr_at_20": 0.9080952380952381, | |
| "mrr_at_50": 0.9084523809523809, | |
| "mrr_at_100": 0.9087981335604287, | |
| "naucs_at_1_max": 0.49321047526673, | |
| "naucs_at_1_std": -0.789039767216295, | |
| "naucs_at_1_diff1": 0.8953858944159624, | |
| "naucs_at_3_max": 0.711251167133517, | |
| "naucs_at_3_std": -0.4164332399626471, | |
| "naucs_at_3_diff1": 0.8885387488328684, | |
| "naucs_at_5_max": 0.7112511671335194, | |
| "naucs_at_5_std": -0.41643323996264636, | |
| "naucs_at_5_diff1": 0.8885387488328625, | |
| "naucs_at_10_max": 0.7075941487706191, | |
| "naucs_at_10_std": -0.5961718020541499, | |
| "naucs_at_10_diff1": 0.8513849984438217, | |
| "naucs_at_20_max": 0.7075941487706191, | |
| "naucs_at_20_std": -0.5961718020541499, | |
| "naucs_at_20_diff1": 0.8513849984438217, | |
| "naucs_at_50_max": 0.561391223155939, | |
| "naucs_at_50_std": -0.8085901027077392, | |
| "naucs_at_50_diff1": 0.7770774976657274, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_government_reports_test": { | |
| "ndcg_at_1": 0.75, | |
| "ndcg_at_3": 0.87226, | |
| "ndcg_at_5": 0.87656, | |
| "ndcg_at_10": 0.87958, | |
| "ndcg_at_20": 0.88734, | |
| "ndcg_at_50": 0.88734, | |
| "ndcg_at_100": 0.88734, | |
| "map_at_1": 0.75, | |
| "map_at_3": 0.845, | |
| "map_at_5": 0.8475, | |
| "map_at_10": 0.84861, | |
| "map_at_20": 0.85085, | |
| "map_at_50": 0.85085, | |
| "map_at_100": 0.85085, | |
| "recall_at_1": 0.75, | |
| "recall_at_3": 0.95, | |
| "recall_at_5": 0.96, | |
| "recall_at_10": 0.97, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.75, | |
| "precision_at_3": 0.31667, | |
| "precision_at_5": 0.192, | |
| "precision_at_10": 0.097, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.75, | |
| "mrr_at_3": 0.8449999999999999, | |
| "mrr_at_5": 0.8474999999999998, | |
| "mrr_at_10": 0.848611111111111, | |
| "mrr_at_20": 0.8508535353535354, | |
| "mrr_at_50": 0.8508535353535354, | |
| "mrr_at_100": 0.8508535353535354, | |
| "naucs_at_1_max": 0.26982731554160105, | |
| "naucs_at_1_std": 0.0717948717948719, | |
| "naucs_at_1_diff1": 0.8517216117216112, | |
| "naucs_at_3_max": 0.7134453781512629, | |
| "naucs_at_3_std": -0.29103641456582646, | |
| "naucs_at_3_diff1": 0.7735760971055052, | |
| "naucs_at_5_max": 0.6418067226890778, | |
| "naucs_at_5_std": -0.6137955182072738, | |
| "naucs_at_5_diff1": 0.7864145658263278, | |
| "naucs_at_10_max": 0.5659819483348872, | |
| "naucs_at_10_std": -0.6305633364456902, | |
| "naucs_at_10_diff1": 0.807812013694364, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_healthcare_industry_test": { | |
| "ndcg_at_1": 0.86, | |
| "ndcg_at_3": 0.91917, | |
| "ndcg_at_5": 0.92734, | |
| "ndcg_at_10": 0.93365, | |
| "ndcg_at_20": 0.93365, | |
| "ndcg_at_50": 0.93365, | |
| "ndcg_at_100": 0.93365, | |
| "map_at_1": 0.86, | |
| "map_at_3": 0.905, | |
| "map_at_5": 0.9095, | |
| "map_at_10": 0.912, | |
| "map_at_20": 0.912, | |
| "map_at_50": 0.912, | |
| "map_at_100": 0.912, | |
| "recall_at_1": 0.86, | |
| "recall_at_3": 0.96, | |
| "recall_at_5": 0.98, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.86, | |
| "precision_at_3": 0.32, | |
| "precision_at_5": 0.196, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.86, | |
| "mrr_at_3": 0.9049999999999998, | |
| "mrr_at_5": 0.9094999999999999, | |
| "mrr_at_10": 0.9119999999999999, | |
| "mrr_at_20": 0.9119999999999999, | |
| "mrr_at_50": 0.9119999999999999, | |
| "mrr_at_100": 0.9119999999999999, | |
| "naucs_at_1_max": 0.14736039905778067, | |
| "naucs_at_1_std": -0.5899958431481216, | |
| "naucs_at_1_diff1": 0.8188998198697522, | |
| "naucs_at_3_max": 0.21171802054154468, | |
| "naucs_at_3_std": -1.2987861811391213, | |
| "naucs_at_3_diff1": 0.8651960784313714, | |
| "naucs_at_5_max": -0.21475256769374024, | |
| "naucs_at_5_std": -1.7399626517273497, | |
| "naucs_at_5_diff1": 0.9346405228758136, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| } | |
| } | |
| } |