{ "metadata": { "timestamp": "2025-03-13T15:23:57.734843", "vidore_benchmark_version": "5.0.1.dev5+g1548c2d.d20250313" }, "metrics": { "vidore/restaurant_esg_reports_beir": { "ndcg_at_1": 0.48718, "ndcg_at_3": 0.51987, "ndcg_at_5": 0.53707, "ndcg_at_10": 0.58691, "ndcg_at_20": 0.60769, "ndcg_at_50": 0.6397, "ndcg_at_100": 0.65482, "map_at_1": 0.36186, "map_at_3": 0.4602, "map_at_5": 0.48616, "map_at_10": 0.51598, "map_at_20": 0.52449, "map_at_50": 0.53241, "map_at_100": 0.5362, "recall_at_1": 0.36186, "recall_at_3": 0.52821, "recall_at_5": 0.59359, "recall_at_10": 0.70563, "recall_at_20": 0.76315, "recall_at_50": 0.88615, "recall_at_100": 0.94866, "precision_at_1": 0.5, "precision_at_3": 0.27564, "precision_at_5": 0.19231, "precision_at_10": 0.13077, "precision_at_20": 0.07596, "precision_at_50": 0.03808, "precision_at_100": 0.02192, "mrr_at_1": 0.5, "mrr_at_3": 0.5769230769230769, "mrr_at_5": 0.5865384615384615, "mrr_at_10": 0.6023733211233211, "mrr_at_20": 0.6069520757020757, "mrr_at_50": 0.6103908065114961, "mrr_at_100": 0.6106960568167464, "naucs_at_1_max": 0.015424312502453706, "naucs_at_1_std": -0.13924671268887046, "naucs_at_1_diff1": 0.774635283388261, "naucs_at_3_max": 0.08232812603721668, "naucs_at_3_std": 0.14582513042567308, "naucs_at_3_diff1": 0.3413749493398797, "naucs_at_5_max": 0.04733940764311593, "naucs_at_5_std": 0.201486754820981, "naucs_at_5_diff1": 0.1679332256651539, "naucs_at_10_max": 0.15213696092763468, "naucs_at_10_std": 0.25898837879862296, "naucs_at_10_diff1": -0.0032105086289669817, "naucs_at_20_max": 0.1701364069679977, "naucs_at_20_std": 0.3491634058276955, "naucs_at_20_diff1": -0.10811392645442519, "naucs_at_50_max": 0.22116549189926218, "naucs_at_50_std": 0.40046893166130704, "naucs_at_50_diff1": -0.22625061791645093, "naucs_at_100_max": 0.23011945384723256, "naucs_at_100_std": 0.42936675975120925, "naucs_at_100_diff1": -0.2960643771988182 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { "ndcg_at_1": 0.54375, "ndcg_at_3": 0.52916, "ndcg_at_5": 0.54779, "ndcg_at_10": 0.58771, "ndcg_at_20": 0.61859, "ndcg_at_50": 0.64913, "ndcg_at_100": 0.66185, "map_at_1": 0.33558, "map_at_3": 0.4307, "map_at_5": 0.46539, "map_at_10": 0.49662, "map_at_20": 0.51175, "map_at_50": 0.52171, "map_at_100": 0.52497, "recall_at_1": 0.33558, "recall_at_3": 0.49142, "recall_at_5": 0.5656, "recall_at_10": 0.67964, "recall_at_20": 0.77075, "recall_at_50": 0.87085, "recall_at_100": 0.91455, "precision_at_1": 0.54375, "precision_at_3": 0.32292, "precision_at_5": 0.24625, "precision_at_10": 0.16063, "precision_at_20": 0.09688, "precision_at_50": 0.04762, "precision_at_100": 0.02675, "mrr_at_1": 0.54375, "mrr_at_3": 0.6177083333333331, "mrr_at_5": 0.6317708333333332, "mrr_at_10": 0.6458953373015872, "mrr_at_20": 0.6501168388392652, "mrr_at_50": 0.6516168070330217, "mrr_at_100": 0.6516168070330217, "naucs_at_1_max": 0.17394147030363272, "naucs_at_1_std": -0.36673902241284584, "naucs_at_1_diff1": 0.43317554550130327, "naucs_at_3_max": 0.10230735338450472, "naucs_at_3_std": -0.16796141270319767, "naucs_at_3_diff1": -0.0021937249263731763, "naucs_at_5_max": 0.06165269158937461, "naucs_at_5_std": -0.16213659251057302, "naucs_at_5_diff1": -0.09605836244564413, "naucs_at_10_max": 0.04062826287163538, "naucs_at_10_std": -0.10386324001578753, "naucs_at_10_diff1": -0.19533141678260185, "naucs_at_20_max": -0.023024941185161928, "naucs_at_20_std": -0.0808216300079185, "naucs_at_20_diff1": -0.2571645591830793, "naucs_at_50_max": -0.06194972750127154, "naucs_at_50_std": 0.023927949244181214, "naucs_at_50_diff1": -0.28301815335673003, "naucs_at_100_max": -0.10101655678389299, "naucs_at_100_std": 0.08336119059224785, "naucs_at_100_diff1": -0.2955701804119539 }, "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { "ndcg_at_1": 0.63793, "ndcg_at_3": 0.61396, "ndcg_at_5": 0.59648, "ndcg_at_10": 0.55957, "ndcg_at_20": 0.58035, "ndcg_at_50": 0.64249, "ndcg_at_100": 0.68124, "map_at_1": 0.07146, "map_at_3": 0.16727, "map_at_5": 0.22845, "map_at_10": 0.29412, "map_at_20": 0.35326, "map_at_50": 0.41154, "map_at_100": 0.43875, "recall_at_1": 0.07146, "recall_at_3": 0.2114, "recall_at_5": 0.31582, "recall_at_10": 0.44705, "recall_at_20": 0.59174, "recall_at_50": 0.77882, "recall_at_100": 0.90011, "precision_at_1": 0.63793, "precision_at_3": 0.58046, "precision_at_5": 0.53448, "precision_at_10": 0.4069, "precision_at_20": 0.31121, "precision_at_50": 0.19621, "precision_at_100": 0.1269, "mrr_at_1": 0.6379310344827587, "mrr_at_3": 0.735632183908046, "mrr_at_5": 0.7502873563218392, "mrr_at_10": 0.7584975369458129, "mrr_at_20": 0.7607245484400658, "mrr_at_50": 0.7607245484400658, "mrr_at_100": 0.7607245484400658, "naucs_at_1_max": 0.30951896933248446, "naucs_at_1_std": 0.0009385908345415833, "naucs_at_1_diff1": 0.12207984020363619, "naucs_at_3_max": 0.3174256751774014, "naucs_at_3_std": 0.07401442252929685, "naucs_at_3_diff1": 0.0813575535006513, "naucs_at_5_max": 0.3929619603728344, "naucs_at_5_std": 0.1689559171748497, "naucs_at_5_diff1": 0.04881016825842116, "naucs_at_10_max": 0.4375324160420264, "naucs_at_10_std": 0.2657788992938007, "naucs_at_10_diff1": -0.08210009736139143, "naucs_at_20_max": 0.435316184966006, "naucs_at_20_std": 0.3991338622440554, "naucs_at_20_diff1": -0.0429704940007693, "naucs_at_50_max": 0.3483632241805499, "naucs_at_50_std": 0.33986582587288133, "naucs_at_50_diff1": 0.0168280192203874, "naucs_at_100_max": 0.28796239161632026, "naucs_at_100_std": 0.27465381207019524, "naucs_at_100_diff1": 0.04462948923164142 }, "vidore/synthetic_rse_restaurant_filtered_v1.0": { "ndcg_at_1": 0.42105, "ndcg_at_3": 0.40524, "ndcg_at_5": 0.45864, "ndcg_at_10": 0.50339, "ndcg_at_20": 0.52015, "ndcg_at_50": 0.55124, "ndcg_at_100": 0.58182, "map_at_1": 0.20117, "map_at_3": 0.29694, "map_at_5": 0.34743, "map_at_10": 0.38591, "map_at_20": 0.39764, "map_at_50": 0.40953, "map_at_100": 0.41648, "recall_at_1": 0.20117, "recall_at_3": 0.35925, "recall_at_5": 0.49808, "recall_at_10": 0.6391, "recall_at_20": 0.69492, "recall_at_50": 0.79371, "recall_at_100": 0.91345, "precision_at_1": 0.42105, "precision_at_3": 0.26901, "precision_at_5": 0.24561, "precision_at_10": 0.16842, "precision_at_20": 0.09912, "precision_at_50": 0.05298, "precision_at_100": 0.03316, "mrr_at_1": 0.42105263157894735, "mrr_at_3": 0.5058479532163742, "mrr_at_5": 0.5330409356725145, "mrr_at_10": 0.5501461988304093, "mrr_at_20": 0.5501461988304093, "mrr_at_50": 0.5521026864735273, "mrr_at_100": 0.5529842660756151, "naucs_at_1_max": 0.2681727937415602, "naucs_at_1_std": 0.26125468443050986, "naucs_at_1_diff1": 0.1925487726366543, "naucs_at_3_max": 0.15154876755279928, "naucs_at_3_std": 0.187284520490265, "naucs_at_3_diff1": 0.13314931121987555, "naucs_at_5_max": 0.06746755549674449, "naucs_at_5_std": 0.17211166986401807, "naucs_at_5_diff1": 0.09922461829032554, "naucs_at_10_max": 0.1173297532733523, "naucs_at_10_std": 0.2334653355769548, "naucs_at_10_diff1": 0.13391976394738275, "naucs_at_20_max": -0.004406787213674995, "naucs_at_20_std": 0.2228534925354155, "naucs_at_20_diff1": 0.0419546501642916, "naucs_at_50_max": -0.09092484597679043, "naucs_at_50_std": 0.2015080134456558, "naucs_at_50_diff1": -0.06928136029689304, "naucs_at_100_max": -0.14425251784397497, "naucs_at_100_std": 0.2117646090542283, "naucs_at_100_diff1": -0.12598719754364754 }, "vidore/synthetic_axa_filtered_v1.0": { "ndcg_at_1": 0.38889, "ndcg_at_3": 0.5418, "ndcg_at_5": 0.50473, "ndcg_at_10": 0.53634, "ndcg_at_20": 0.56481, "ndcg_at_50": 0.60088, "ndcg_at_100": 0.62788, "map_at_1": 0.19085, "map_at_3": 0.33844, "map_at_5": 0.34887, "map_at_10": 0.3905, "map_at_20": 0.41424, "map_at_50": 0.42936, "map_at_100": 0.43807, "recall_at_1": 0.19085, "recall_at_3": 0.52023, "recall_at_5": 0.53824, "recall_at_10": 0.68819, "recall_at_20": 0.76306, "recall_at_50": 0.85789, "recall_at_100": 0.93331, "precision_at_1": 0.38889, "precision_at_3": 0.38889, "precision_at_5": 0.26667, "precision_at_10": 0.2, "precision_at_20": 0.13333, "precision_at_50": 0.06889, "precision_at_100": 0.04222, "mrr_at_1": 0.3888888888888889, "mrr_at_3": 0.5740740740740741, "mrr_at_5": 0.587962962962963, "mrr_at_10": 0.6033950617283952, "mrr_at_20": 0.6033950617283952, "mrr_at_50": 0.6033950617283952, "mrr_at_100": 0.6033950617283952, "naucs_at_1_max": 0.14105650005960155, "naucs_at_1_std": -0.01993661989253736, "naucs_at_1_diff1": 0.43542988491270046, "naucs_at_3_max": -0.22826796699446616, "naucs_at_3_std": -0.004572665724137645, "naucs_at_3_diff1": 0.07496811347896108, "naucs_at_5_max": -0.3852842673944422, "naucs_at_5_std": -0.15791921982494983, "naucs_at_5_diff1": 0.2052446052071864, "naucs_at_10_max": -0.3499853580695824, "naucs_at_10_std": -0.07324394708860461, "naucs_at_10_diff1": 0.06796721460372077, "naucs_at_20_max": -0.360118573152839, "naucs_at_20_std": -0.15283583044728602, "naucs_at_20_diff1": 0.26460065509124203, "naucs_at_50_max": -0.38416433266363276, "naucs_at_50_std": -0.1571206204390072, "naucs_at_50_diff1": 0.4313242188093156, "naucs_at_100_max": -0.4149515845406916, "naucs_at_100_std": -0.19401302943385454, "naucs_at_100_diff1": 0.49450789205035006 }, "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { "ndcg_at_1": 0.39912, "ndcg_at_3": 0.43071, "ndcg_at_5": 0.46444, "ndcg_at_10": 0.4968, "ndcg_at_20": 0.52854, "ndcg_at_50": 0.56102, "ndcg_at_100": 0.58233, "map_at_1": 0.17982, "map_at_3": 0.30157, "map_at_5": 0.34561, "map_at_10": 0.38009, "map_at_20": 0.39814, "map_at_50": 0.41046, "map_at_100": 0.41606, "recall_at_1": 0.17982, "recall_at_3": 0.4008, "recall_at_5": 0.50897, "recall_at_10": 0.62213, "recall_at_20": 0.72976, "recall_at_50": 0.83655, "recall_at_100": 0.915, "precision_at_1": 0.39912, "precision_at_3": 0.30263, "precision_at_5": 0.25175, "precision_at_10": 0.16886, "precision_at_20": 0.10636, "precision_at_50": 0.05632, "precision_at_100": 0.03329, "mrr_at_1": 0.3991228070175439, "mrr_at_3": 0.5160818713450291, "mrr_at_5": 0.5342836257309941, "mrr_at_10": 0.5459934558618768, "mrr_at_20": 0.5499615268119828, "mrr_at_50": 0.5520170407199186, "mrr_at_100": 0.5523559958857708, "naucs_at_1_max": 0.15755765375477634, "naucs_at_1_std": 0.1937615823895248, "naucs_at_1_diff1": 0.20176410519897442, "naucs_at_3_max": 0.21637882247590812, "naucs_at_3_std": 0.21919003966069206, "naucs_at_3_diff1": 0.09426673538244577, "naucs_at_5_max": 0.17022232642187712, "naucs_at_5_std": 0.20399998951672477, "naucs_at_5_diff1": 0.07433346087483172, "naucs_at_10_max": 0.17445157988680682, "naucs_at_10_std": 0.19671403249308644, "naucs_at_10_diff1": 0.042552504976369365, "naucs_at_20_max": 0.0564754707494715, "naucs_at_20_std": 0.14487480859508034, "naucs_at_20_diff1": -0.050996649070789594, "naucs_at_50_max": -0.032342844885820655, "naucs_at_50_std": 0.0919650425307495, "naucs_at_50_diff1": -0.1049317890928347, "naucs_at_100_max": -0.0813860336462654, "naucs_at_100_std": 0.07405564056158524, "naucs_at_100_diff1": -0.15314098265443818 }, "vidore/synthetic_axa_filtered_v1.0_multilingual": { "ndcg_at_1": 0.40278, "ndcg_at_3": 0.45413, "ndcg_at_5": 0.45175, "ndcg_at_10": 0.49692, "ndcg_at_20": 0.53744, "ndcg_at_50": 0.57505, "ndcg_at_100": 0.60205, "map_at_1": 0.20688, "map_at_3": 0.30245, "map_at_5": 0.32956, "map_at_10": 0.37191, "map_at_20": 0.39679, "map_at_50": 0.41224, "map_at_100": 0.42049, "recall_at_1": 0.20688, "recall_at_3": 0.39577, "recall_at_5": 0.462, "recall_at_10": 0.6319, "recall_at_20": 0.73905, "recall_at_50": 0.83135, "recall_at_100": 0.90763, "precision_at_1": 0.40278, "precision_at_3": 0.30556, "precision_at_5": 0.23889, "precision_at_10": 0.17917, "precision_at_20": 0.125, "precision_at_50": 0.06639, "precision_at_100": 0.04097, "mrr_at_1": 0.4027777777777778, "mrr_at_3": 0.5300925925925926, "mrr_at_5": 0.5564814814814815, "mrr_at_10": 0.5736221340388007, "mrr_at_20": 0.5763117283950616, "mrr_at_50": 0.5763117283950616, "mrr_at_100": 0.5763117283950616, "naucs_at_1_max": 0.20656968495031644, "naucs_at_1_std": 0.0817297457698203, "naucs_at_1_diff1": 0.2238136061191017, "naucs_at_3_max": 0.25163264129737517, "naucs_at_3_std": 0.17030727986741342, "naucs_at_3_diff1": 0.03578498764457108, "naucs_at_5_max": 0.08903441500332349, "naucs_at_5_std": 0.04568596558397242, "naucs_at_5_diff1": -0.009941530139102166, "naucs_at_10_max": -0.020546119491255295, "naucs_at_10_std": 0.008887544287522966, "naucs_at_10_diff1": -0.09432640328513163, "naucs_at_20_max": -0.1319344233352006, "naucs_at_20_std": -0.044221131984674825, "naucs_at_20_diff1": -0.06538820564150717, "naucs_at_50_max": -0.18743175691838732, "naucs_at_50_std": -0.0972663161231581, "naucs_at_50_diff1": -0.017105606627328275, "naucs_at_100_max": -0.2325355500441028, "naucs_at_100_std": -0.15406419774878136, "naucs_at_100_diff1": -0.018290038816855033 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { "ndcg_at_1": 0.43438, "ndcg_at_3": 0.45593, "ndcg_at_5": 0.47695, "ndcg_at_10": 0.5166, "ndcg_at_20": 0.54502, "ndcg_at_50": 0.57616, "ndcg_at_100": 0.59352, "map_at_1": 0.27154, "map_at_3": 0.36745, "map_at_5": 0.39714, "map_at_10": 0.42629, "map_at_20": 0.43946, "map_at_50": 0.44896, "map_at_100": 0.45239, "recall_at_1": 0.27154, "recall_at_3": 0.44101, "recall_at_5": 0.51211, "recall_at_10": 0.62263, "recall_at_20": 0.70666, "recall_at_50": 0.81063, "recall_at_100": 0.87902, "precision_at_1": 0.43438, "precision_at_3": 0.27917, "precision_at_5": 0.21531, "precision_at_10": 0.14359, "precision_at_20": 0.08672, "precision_at_50": 0.04341, "precision_at_100": 0.02502, "mrr_at_1": 0.434375, "mrr_at_3": 0.5317708333333331, "mrr_at_5": 0.5496614583333328, "mrr_at_10": 0.5630952380952378, "mrr_at_20": 0.5669521784558423, "mrr_at_50": 0.5688358670627492, "mrr_at_100": 0.5692193060123717, "naucs_at_1_max": 0.2666912240570166, "naucs_at_1_std": 0.05389058257999144, "naucs_at_1_diff1": 0.4321239157776483, "naucs_at_3_max": 0.17727410640755303, "naucs_at_3_std": 0.01577475274662206, "naucs_at_3_diff1": 0.09089286579349252, "naucs_at_5_max": 0.13070625524250945, "naucs_at_5_std": 0.01198523787261333, "naucs_at_5_diff1": -0.013247813819131482, "naucs_at_10_max": 0.08901372928565265, "naucs_at_10_std": 0.0242424061104149, "naucs_at_10_diff1": -0.0824662153065525, "naucs_at_20_max": 0.046509940234363965, "naucs_at_20_std": 0.023421019939509093, "naucs_at_20_diff1": -0.13765222771211638, "naucs_at_50_max": -0.016368222256068723, "naucs_at_50_std": 0.017686339762030524, "naucs_at_50_diff1": -0.18185950361763814, "naucs_at_100_max": -0.05809760099241634, "naucs_at_100_std": -0.0007126993337199372, "naucs_at_100_diff1": -0.20245463094499655 }, "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { "ndcg_at_1": 0.52586, "ndcg_at_3": 0.49942, "ndcg_at_5": 0.48681, "ndcg_at_10": 0.46438, "ndcg_at_20": 0.49413, "ndcg_at_50": 0.56018, "ndcg_at_100": 0.60792, "map_at_1": 0.05323, "map_at_3": 0.12858, "map_at_5": 0.17351, "map_at_10": 0.22551, "map_at_20": 0.27667, "map_at_50": 0.3298, "map_at_100": 0.35718, "recall_at_1": 0.05323, "recall_at_3": 0.16936, "recall_at_5": 0.25112, "recall_at_10": 0.37154, "recall_at_20": 0.52494, "recall_at_50": 0.7231, "recall_at_100": 0.86824, "precision_at_1": 0.52586, "precision_at_3": 0.4727, "precision_at_5": 0.43966, "precision_at_10": 0.35, "precision_at_20": 0.27651, "precision_at_50": 0.17888, "precision_at_100": 0.12013, "mrr_at_1": 0.5258620689655172, "mrr_at_3": 0.6163793103448275, "mrr_at_5": 0.6372844827586207, "mrr_at_10": 0.6511271893814995, "mrr_at_20": 0.6548831549338646, "mrr_at_50": 0.6554856928251602, "mrr_at_100": 0.6556279827912883, "naucs_at_1_max": 0.1755971141206357, "naucs_at_1_std": 0.18264029674960228, "naucs_at_1_diff1": 0.06925322234490841, "naucs_at_3_max": 0.2812764433128787, "naucs_at_3_std": 0.2690356636124753, "naucs_at_3_diff1": 0.016089000880329198, "naucs_at_5_max": 0.283989396373602, "naucs_at_5_std": 0.29562037646588923, "naucs_at_5_diff1": 0.009509118422791094, "naucs_at_10_max": 0.23668581693373705, "naucs_at_10_std": 0.2844259803726197, "naucs_at_10_diff1": -0.07126374015391594, "naucs_at_20_max": 0.2076351063428368, "naucs_at_20_std": 0.2991176793010035, "naucs_at_20_diff1": -0.08337920885811372, "naucs_at_50_max": 0.1238437265668122, "naucs_at_50_std": 0.2430342456930391, "naucs_at_50_diff1": -0.05232657907650674, "naucs_at_100_max": 0.05825862012096387, "naucs_at_100_std": 0.19337982478846943, "naucs_at_100_diff1": -0.020634764162514304 }, "vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.736, "ndcg_at_3": 0.79238, "ndcg_at_5": 0.80581, "ndcg_at_10": 0.82083, "ndcg_at_20": 0.82784, "ndcg_at_50": 0.83516, "ndcg_at_100": 0.83776, "map_at_1": 0.736, "map_at_3": 0.77867, "map_at_5": 0.78627, "map_at_10": 0.79255, "map_at_20": 0.79443, "map_at_50": 0.79568, "map_at_100": 0.79591, "recall_at_1": 0.736, "recall_at_3": 0.832, "recall_at_5": 0.864, "recall_at_10": 0.91, "recall_at_20": 0.938, "recall_at_50": 0.974, "recall_at_100": 0.99, "precision_at_1": 0.736, "precision_at_3": 0.27733, "precision_at_5": 0.1728, "precision_at_10": 0.091, "precision_at_20": 0.0469, "precision_at_50": 0.01948, "precision_at_100": 0.0099, "mrr_at_1": 0.736, "mrr_at_3": 0.7786666666666667, "mrr_at_5": 0.7862666666666667, "mrr_at_10": 0.7925476190476194, "mrr_at_20": 0.7944328455428302, "mrr_at_50": 0.7956750456398185, "mrr_at_100": 0.7959050774896889, "naucs_at_1_max": 0.6486039470733347, "naucs_at_1_std": -0.11010636593726965, "naucs_at_1_diff1": 0.8885840835549293, "naucs_at_3_max": 0.6259037134406845, "naucs_at_3_std": -0.18474484765973373, "naucs_at_3_diff1": 0.8376660720153996, "naucs_at_5_max": 0.6215153325368371, "naucs_at_5_std": -0.17638817773226517, "naucs_at_5_diff1": 0.8335253456221203, "naucs_at_10_max": 0.5732648615001562, "naucs_at_10_std": -0.2490196078431399, "naucs_at_10_diff1": 0.787851436871045, "naucs_at_20_max": 0.5872112285774529, "naucs_at_20_std": -0.18579259660853667, "naucs_at_20_diff1": 0.8300502996897676, "naucs_at_50_max": 0.8495654672125184, "naucs_at_50_std": 0.27152912447029887, "naucs_at_50_diff1": 0.8371399841988043, "naucs_at_100_max": 0.9183006535947666, "naucs_at_100_std": 0.49262371615312944, "naucs_at_100_diff1": 0.8921568627450854 }, "vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.34368, "ndcg_at_3": 0.40479, "ndcg_at_5": 0.42673, "ndcg_at_10": 0.45035, "ndcg_at_20": 0.46746, "ndcg_at_50": 0.48358, "ndcg_at_100": 0.49656, "map_at_1": 0.34368, "map_at_3": 0.38987, "map_at_5": 0.40207, "map_at_10": 0.41179, "map_at_20": 0.41664, "map_at_50": 0.41935, "map_at_100": 0.42051, "recall_at_1": 0.34368, "recall_at_3": 0.44789, "recall_at_5": 0.50111, "recall_at_10": 0.57428, "recall_at_20": 0.6408, "recall_at_50": 0.72062, "recall_at_100": 0.80044, "precision_at_1": 0.34368, "precision_at_3": 0.1493, "precision_at_5": 0.10022, "precision_at_10": 0.05743, "precision_at_20": 0.03204, "precision_at_50": 0.01441, "precision_at_100": 0.008, "mrr_at_1": 0.3436807095343681, "mrr_at_3": 0.3898743532889875, "mrr_at_5": 0.4020694752402069, "mrr_at_10": 0.4117877380072501, "mrr_at_20": 0.4166391095440233, "mrr_at_50": 0.4194072008524503, "mrr_at_100": 0.42066245247914874, "naucs_at_1_max": 0.6937621431789922, "naucs_at_1_std": 0.3966932403411503, "naucs_at_1_diff1": 0.7145229983797862, "naucs_at_3_max": 0.7320406126458378, "naucs_at_3_std": 0.45563802569337475, "naucs_at_3_diff1": 0.6143081963671839, "naucs_at_5_max": 0.6880199080240672, "naucs_at_5_std": 0.44498830503027725, "naucs_at_5_diff1": 0.5636895248104472, "naucs_at_10_max": 0.7395456562744627, "naucs_at_10_std": 0.5421201928794931, "naucs_at_10_diff1": 0.5433914071660855, "naucs_at_20_max": 0.7099358788663228, "naucs_at_20_std": 0.505428632850115, "naucs_at_20_diff1": 0.5026852667645294, "naucs_at_50_max": 0.7044396744042806, "naucs_at_50_std": 0.5292950459939872, "naucs_at_50_diff1": 0.48465037492486385, "naucs_at_100_max": 0.6982871591556156, "naucs_at_100_std": 0.5542470657836935, "naucs_at_100_diff1": 0.4520088087347075 }, "vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.76923, "ndcg_at_3": 0.83858, "ndcg_at_5": 0.85026, "ndcg_at_10": 0.86321, "ndcg_at_20": 0.86616, "ndcg_at_50": 0.86899, "ndcg_at_100": 0.86929, "map_at_1": 0.76923, "map_at_3": 0.82254, "map_at_5": 0.82901, "map_at_10": 0.83467, "map_at_20": 0.83541, "map_at_50": 0.83587, "map_at_100": 0.8359, "recall_at_1": 0.76923, "recall_at_3": 0.88462, "recall_at_5": 0.91296, "recall_at_10": 0.95142, "recall_at_20": 0.96356, "recall_at_50": 0.97773, "recall_at_100": 0.97976, "precision_at_1": 0.76923, "precision_at_3": 0.29487, "precision_at_5": 0.18259, "precision_at_10": 0.09514, "precision_at_20": 0.04818, "precision_at_50": 0.01955, "precision_at_100": 0.0098, "mrr_at_1": 0.7692307692307693, "mrr_at_3": 0.8225371120107962, "mrr_at_5": 0.8290148448043184, "mrr_at_10": 0.8346667951931113, "mrr_at_20": 0.8354146169277593, "mrr_at_50": 0.8358748221328831, "mrr_at_100": 0.8359177231300928, "naucs_at_1_max": 0.5127634005653434, "naucs_at_1_std": -0.03567831485698759, "naucs_at_1_diff1": 0.8320638205788135, "naucs_at_3_max": 0.5021250200120738, "naucs_at_3_std": -0.07399077187048693, "naucs_at_3_diff1": 0.7752634768446268, "naucs_at_5_max": 0.5335848645863028, "naucs_at_5_std": -0.02593966089833383, "naucs_at_5_diff1": 0.7661635495872359, "naucs_at_10_max": 0.9000642537485549, "naucs_at_10_std": 0.4822680430176945, "naucs_at_10_diff1": 0.7890895385688544, "naucs_at_20_max": 0.8894376823215577, "naucs_at_20_std": 0.4867325787808216, "naucs_at_20_diff1": 0.7414713954152902, "naucs_at_50_max": 0.9762542810791424, "naucs_at_50_std": 0.9525085621582665, "naucs_at_50_diff1": 0.733736269334063, "naucs_at_100_max": 0.9738797091870466, "naucs_at_100_std": 0.960819563780575, "naucs_at_100_diff1": 0.7348833700432545 }, "vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.69286, "ndcg_at_3": 0.75259, "ndcg_at_5": 0.76889, "ndcg_at_10": 0.79021, "ndcg_at_20": 0.80681, "ndcg_at_50": 0.81245, "ndcg_at_100": 0.81307, "map_at_1": 0.69286, "map_at_3": 0.7375, "map_at_5": 0.74661, "map_at_10": 0.75505, "map_at_20": 0.75979, "map_at_50": 0.76068, "map_at_100": 0.76074, "recall_at_1": 0.69286, "recall_at_3": 0.79643, "recall_at_5": 0.83571, "recall_at_10": 0.90357, "recall_at_20": 0.96786, "recall_at_50": 0.99643, "recall_at_100": 1.0, "precision_at_1": 0.69286, "precision_at_3": 0.26548, "precision_at_5": 0.16714, "precision_at_10": 0.09036, "precision_at_20": 0.04839, "precision_at_50": 0.01993, "precision_at_100": 0.01, "mrr_at_1": 0.6928571428571428, "mrr_at_3": 0.7375000000000002, "mrr_at_5": 0.746607142857143, "mrr_at_10": 0.7550481859410433, "mrr_at_20": 0.7597886651746948, "mrr_at_50": 0.7606781401931197, "mrr_at_100": 0.7607430752580548, "naucs_at_1_max": 0.6018301415698155, "naucs_at_1_std": 0.32571196816263426, "naucs_at_1_diff1": 0.7888873398309274, "naucs_at_3_max": 0.6444276300161007, "naucs_at_3_std": 0.3241186355677946, "naucs_at_3_diff1": 0.7616114869917906, "naucs_at_5_max": 0.6430603688016092, "naucs_at_5_std": 0.32898001968082796, "naucs_at_5_diff1": 0.725377572412612, "naucs_at_10_max": 0.5787598990213383, "naucs_at_10_std": 0.39082546598886503, "naucs_at_10_diff1": 0.6965107030466515, "naucs_at_20_max": 0.5347546425977789, "naucs_at_20_std": 0.10488639900404371, "naucs_at_20_diff1": 0.5825293080195045, "naucs_at_50_max": 0.8692810457515607, "naucs_at_50_std": 0.5541549953314449, "naucs_at_50_diff1": 0.8692810457515607, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/tatdqa_test": { "ndcg_at_1": 0.35115, "ndcg_at_3": 0.45665, "ndcg_at_5": 0.49299, "ndcg_at_10": 0.53515, "ndcg_at_20": 0.55722, "ndcg_at_50": 0.57662, "ndcg_at_100": 0.58344, "map_at_1": 0.35115, "map_at_3": 0.43074, "map_at_5": 0.45094, "map_at_10": 0.46853, "map_at_20": 0.47464, "map_at_50": 0.47794, "map_at_100": 0.47855, "recall_at_1": 0.35115, "recall_at_3": 0.53159, "recall_at_5": 0.61968, "recall_at_10": 0.74909, "recall_at_20": 0.83597, "recall_at_50": 0.93135, "recall_at_100": 0.97327, "precision_at_1": 0.35115, "precision_at_3": 0.1772, "precision_at_5": 0.12394, "precision_at_10": 0.07491, "precision_at_20": 0.0418, "precision_at_50": 0.01863, "precision_at_100": 0.00973, "mrr_at_1": 0.35297691373025514, "mrr_at_3": 0.431652490886999, "mrr_at_5": 0.45188335358444764, "mrr_at_10": 0.4694830179945614, "mrr_at_20": 0.47559566338222015, "mrr_at_50": 0.47890243738258254, "mrr_at_100": 0.47950989875566846, "naucs_at_1_max": 0.16045300557832828, "naucs_at_1_std": -0.18712590459385856, "naucs_at_1_diff1": 0.5440702710166525, "naucs_at_3_max": 0.20647987331228912, "naucs_at_3_std": -0.12049997692079921, "naucs_at_3_diff1": 0.4450522896579701, "naucs_at_5_max": 0.2514186053554693, "naucs_at_5_std": -0.07250138069379193, "naucs_at_5_diff1": 0.43314416061424443, "naucs_at_10_max": 0.31938596426902055, "naucs_at_10_std": -0.03160768541250311, "naucs_at_10_diff1": 0.4063667061259836, "naucs_at_20_max": 0.37555947710006393, "naucs_at_20_std": 0.07079816177576714, "naucs_at_20_diff1": 0.4080320503888041, "naucs_at_50_max": 0.37314614111817646, "naucs_at_50_std": 0.15009417772189595, "naucs_at_50_diff1": 0.3940999242861563, "naucs_at_100_max": 0.5170529950663407, "naucs_at_100_std": 0.41066786010599293, "naucs_at_100_diff1": 0.41541021771983067 }, "vidore/shiftproject_test": { "ndcg_at_1": 0.44, "ndcg_at_3": 0.59095, "ndcg_at_5": 0.61204, "ndcg_at_10": 0.6477, "ndcg_at_20": 0.6722, "ndcg_at_50": 0.67625, "ndcg_at_100": 0.678, "map_at_1": 0.44, "map_at_3": 0.55333, "map_at_5": 0.56533, "map_at_10": 0.58014, "map_at_20": 0.58646, "map_at_50": 0.58714, "map_at_100": 0.58733, "recall_at_1": 0.44, "recall_at_3": 0.7, "recall_at_5": 0.75, "recall_at_10": 0.86, "recall_at_20": 0.96, "recall_at_50": 0.98, "recall_at_100": 0.99, "precision_at_1": 0.44, "precision_at_3": 0.23333, "precision_at_5": 0.15, "precision_at_10": 0.086, "precision_at_20": 0.048, "precision_at_50": 0.0196, "precision_at_100": 0.0099, "mrr_at_1": 0.44, "mrr_at_3": 0.5533333333333333, "mrr_at_5": 0.5653333333333334, "mrr_at_10": 0.580138888888889, "mrr_at_20": 0.5864581566094724, "mrr_at_50": 0.5871363175290126, "mrr_at_100": 0.5873286252213203, "naucs_at_1_max": -0.004671921490838898, "naucs_at_1_std": -0.12310121626559477, "naucs_at_1_diff1": 0.3854335229942058, "naucs_at_3_max": 0.06802377949918934, "naucs_at_3_std": -0.11448387677895848, "naucs_at_3_diff1": 0.31131327688704724, "naucs_at_5_max": -0.021580324437466547, "naucs_at_5_std": -0.21312401883830379, "naucs_at_5_diff1": 0.41149136577707984, "naucs_at_10_max": -0.1736871276153512, "naucs_at_10_std": -0.35710821671054294, "naucs_at_10_diff1": 0.2720659553831241, "naucs_at_20_max": 0.22934173669468105, "naucs_at_20_std": -0.1219654528478032, "naucs_at_20_diff1": 0.7864145658263278, "naucs_at_50_max": 0.6790382819794637, "naucs_at_50_std": 0.613678804855281, "naucs_at_50_diff1": 0.795751633986937, "naucs_at_100_max": 0.35807656395892007, "naucs_at_100_std": 0.35807656395892007, "naucs_at_100_diff1": 0.8692810457516374 }, "vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.88, "ndcg_at_3": 0.93547, "ndcg_at_5": 0.94365, "ndcg_at_10": 0.94365, "ndcg_at_20": 0.94644, "ndcg_at_50": 0.94644, "ndcg_at_100": 0.94644, "map_at_1": 0.88, "map_at_3": 0.92333, "map_at_5": 0.92783, "map_at_10": 0.92783, "map_at_20": 0.92874, "map_at_50": 0.92874, "map_at_100": 0.92874, "recall_at_1": 0.88, "recall_at_3": 0.97, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.88, "precision_at_3": 0.32333, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.88, "mrr_at_3": 0.9233333333333335, "mrr_at_5": 0.9278333333333335, "mrr_at_10": 0.9278333333333335, "mrr_at_20": 0.9287424242424244, "mrr_at_50": 0.9287424242424244, "mrr_at_100": 0.9287424242424244, "naucs_at_1_max": 0.33646197900095476, "naucs_at_1_std": -0.10101813553929312, "naucs_at_1_diff1": 0.8668071905822462, "naucs_at_3_max": 0.8513849984438244, "naucs_at_3_std": 0.7424525365701778, "naucs_at_3_diff1": 0.664021164021167, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 0.8692810457516413, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 0.8692810457516413, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.86, "ndcg_at_3": 0.92047, "ndcg_at_5": 0.92047, "ndcg_at_10": 0.92381, "ndcg_at_20": 0.92381, "ndcg_at_50": 0.92587, "ndcg_at_100": 0.92927, "map_at_1": 0.86, "map_at_3": 0.90667, "map_at_5": 0.90667, "map_at_10": 0.9081, "map_at_20": 0.9081, "map_at_50": 0.90845, "map_at_100": 0.9088, "recall_at_1": 0.86, "recall_at_3": 0.96, "recall_at_5": 0.96, "recall_at_10": 0.97, "recall_at_20": 0.97, "recall_at_50": 0.98, "recall_at_100": 1.0, "precision_at_1": 0.86, "precision_at_3": 0.32, "precision_at_5": 0.192, "precision_at_10": 0.097, "precision_at_20": 0.0485, "precision_at_50": 0.0196, "precision_at_100": 0.01, "mrr_at_1": 0.86, "mrr_at_3": 0.9066666666666667, "mrr_at_5": 0.9066666666666667, "mrr_at_10": 0.9080952380952381, "mrr_at_20": 0.9080952380952381, "mrr_at_50": 0.9084523809523809, "mrr_at_100": 0.9087981335604287, "naucs_at_1_max": 0.49321047526673, "naucs_at_1_std": -0.789039767216295, "naucs_at_1_diff1": 0.8953858944159624, "naucs_at_3_max": 0.711251167133517, "naucs_at_3_std": -0.4164332399626471, "naucs_at_3_diff1": 0.8885387488328684, "naucs_at_5_max": 0.7112511671335194, "naucs_at_5_std": -0.41643323996264636, "naucs_at_5_diff1": 0.8885387488328625, "naucs_at_10_max": 0.7075941487706191, "naucs_at_10_std": -0.5961718020541499, "naucs_at_10_diff1": 0.8513849984438217, "naucs_at_20_max": 0.7075941487706191, "naucs_at_20_std": -0.5961718020541499, "naucs_at_20_diff1": 0.8513849984438217, "naucs_at_50_max": 0.561391223155939, "naucs_at_50_std": -0.8085901027077392, "naucs_at_50_diff1": 0.7770774976657274, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.75, "ndcg_at_3": 0.87226, "ndcg_at_5": 0.87656, "ndcg_at_10": 0.87958, "ndcg_at_20": 0.88734, "ndcg_at_50": 0.88734, "ndcg_at_100": 0.88734, "map_at_1": 0.75, "map_at_3": 0.845, "map_at_5": 0.8475, "map_at_10": 0.84861, "map_at_20": 0.85085, "map_at_50": 0.85085, "map_at_100": 0.85085, "recall_at_1": 0.75, "recall_at_3": 0.95, "recall_at_5": 0.96, "recall_at_10": 0.97, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.75, "precision_at_3": 0.31667, "precision_at_5": 0.192, "precision_at_10": 0.097, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.75, "mrr_at_3": 0.8449999999999999, "mrr_at_5": 0.8474999999999998, "mrr_at_10": 0.848611111111111, "mrr_at_20": 0.8508535353535354, "mrr_at_50": 0.8508535353535354, "mrr_at_100": 0.8508535353535354, "naucs_at_1_max": 0.26982731554160105, "naucs_at_1_std": 0.0717948717948719, "naucs_at_1_diff1": 0.8517216117216112, "naucs_at_3_max": 0.7134453781512629, "naucs_at_3_std": -0.29103641456582646, "naucs_at_3_diff1": 0.7735760971055052, "naucs_at_5_max": 0.6418067226890778, "naucs_at_5_std": -0.6137955182072738, "naucs_at_5_diff1": 0.7864145658263278, "naucs_at_10_max": 0.5659819483348872, "naucs_at_10_std": -0.6305633364456902, "naucs_at_10_diff1": 0.807812013694364, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.86, "ndcg_at_3": 0.91917, "ndcg_at_5": 0.92734, "ndcg_at_10": 0.93365, "ndcg_at_20": 0.93365, "ndcg_at_50": 0.93365, "ndcg_at_100": 0.93365, "map_at_1": 0.86, "map_at_3": 0.905, "map_at_5": 0.9095, "map_at_10": 0.912, "map_at_20": 0.912, "map_at_50": 0.912, "map_at_100": 0.912, "recall_at_1": 0.86, "recall_at_3": 0.96, "recall_at_5": 0.98, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.86, "precision_at_3": 0.32, "precision_at_5": 0.196, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.86, "mrr_at_3": 0.9049999999999998, "mrr_at_5": 0.9094999999999999, "mrr_at_10": 0.9119999999999999, "mrr_at_20": 0.9119999999999999, "mrr_at_50": 0.9119999999999999, "mrr_at_100": 0.9119999999999999, "naucs_at_1_max": 0.14736039905778067, "naucs_at_1_std": -0.5899958431481216, "naucs_at_1_diff1": 0.8188998198697522, "naucs_at_3_max": 0.21171802054154468, "naucs_at_3_std": -1.2987861811391213, "naucs_at_3_diff1": 0.8651960784313714, "naucs_at_5_max": -0.21475256769374024, "naucs_at_5_std": -1.7399626517273497, "naucs_at_5_diff1": 0.9346405228758136, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null } } }