{ "metadata": { "timestamp": "2025-03-12T10:52:59.228266", "vidore_benchmark_version": "4.0.3.dev37+g1548c2d.d20250311" }, "metrics": { "vidore/restaurant_esg_reports_beir": { "ndcg_at_1": 0.46154, "ndcg_at_3": 0.51553, "ndcg_at_5": 0.56137, "ndcg_at_10": 0.59354, "ndcg_at_20": 0.63362, "ndcg_at_50": 0.65072, "ndcg_at_100": 0.66201, "map_at_1": 0.36346, "map_at_3": 0.44902, "map_at_5": 0.49142, "map_at_10": 0.5106, "map_at_20": 0.52664, "map_at_50": 0.53321, "map_at_100": 0.53551, "recall_at_1": 0.36346, "recall_at_3": 0.50948, "recall_at_5": 0.6369, "recall_at_10": 0.71978, "recall_at_20": 0.83934, "recall_at_50": 0.90182, "recall_at_100": 0.94766, "precision_at_1": 0.46154, "precision_at_3": 0.27564, "precision_at_5": 0.22308, "precision_at_10": 0.13462, "precision_at_20": 0.08462, "precision_at_50": 0.03962, "precision_at_100": 0.02173, "mrr_at_1": 0.46153846153846156, "mrr_at_3": 0.564102564102564, "mrr_at_5": 0.5910256410256409, "mrr_at_10": 0.6018620268620267, "mrr_at_20": 0.609330199714815, "mrr_at_50": 0.6099933296882898, "mrr_at_100": 0.6099933296882898, "naucs_at_1_max": 0.20368019444404006, "naucs_at_1_std": -0.05896301709840663, "naucs_at_1_diff1": 0.5027857837399152, "naucs_at_3_max": 0.2207596691773764, "naucs_at_3_std": 0.006572688085190094, "naucs_at_3_diff1": 0.07535489280712841, "naucs_at_5_max": 0.3055052564988994, "naucs_at_5_std": 0.2695070152185379, "naucs_at_5_diff1": 0.028735419474887416, "naucs_at_10_max": 0.13022275163938632, "naucs_at_10_std": 0.16804625913226817, "naucs_at_10_diff1": -0.1044735223259232, "naucs_at_20_max": 0.030677866275297932, "naucs_at_20_std": 0.2067178669184376, "naucs_at_20_diff1": -0.15547715926288114, "naucs_at_50_max": 0.06721043932684811, "naucs_at_50_std": 0.2814855535601418, "naucs_at_50_diff1": -0.15644474250949775, "naucs_at_100_max": 0.04140512572483361, "naucs_at_100_std": 0.2754882090192102, "naucs_at_100_diff1": -0.20499364042570448 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { "ndcg_at_1": 0.50625, "ndcg_at_3": 0.53801, "ndcg_at_5": 0.56367, "ndcg_at_10": 0.59144, "ndcg_at_20": 0.62281, "ndcg_at_50": 0.64988, "ndcg_at_100": 0.6614, "map_at_1": 0.2947, "map_at_3": 0.42389, "map_at_5": 0.46584, "map_at_10": 0.49641, "map_at_20": 0.51381, "map_at_50": 0.52294, "map_at_100": 0.52512, "recall_at_1": 0.2947, "recall_at_3": 0.51066, "recall_at_5": 0.6115, "recall_at_10": 0.69677, "recall_at_20": 0.78713, "recall_at_50": 0.87152, "recall_at_100": 0.92041, "precision_at_1": 0.50625, "precision_at_3": 0.34583, "precision_at_5": 0.26625, "precision_at_10": 0.16813, "precision_at_20": 0.10125, "precision_at_50": 0.04987, "precision_at_100": 0.02706, "mrr_at_1": 0.50625, "mrr_at_3": 0.6187499999999999, "mrr_at_5": 0.6390625, "mrr_at_10": 0.6450694444444445, "mrr_at_20": 0.6481967685092686, "mrr_at_50": 0.6493743580027906, "mrr_at_100": 0.6497234378933107, "naucs_at_1_max": 0.2982323803776623, "naucs_at_1_std": -0.053774525827026284, "naucs_at_1_diff1": 0.5200020115159291, "naucs_at_3_max": 0.18024374308063665, "naucs_at_3_std": 0.07332710212526489, "naucs_at_3_diff1": 0.019165502096226788, "naucs_at_5_max": 0.14274071990595502, "naucs_at_5_std": 0.09730846621519294, "naucs_at_5_diff1": -0.00690192345139435, "naucs_at_10_max": 0.1869838718265326, "naucs_at_10_std": 0.17477047588380465, "naucs_at_10_diff1": -0.07438077296572282, "naucs_at_20_max": 0.24640900247147313, "naucs_at_20_std": 0.2484156281222067, "naucs_at_20_diff1": -0.1333943313877057, "naucs_at_50_max": 0.22134620285423043, "naucs_at_50_std": 0.3203443552497451, "naucs_at_50_diff1": -0.13630064984709492, "naucs_at_100_max": 0.18852863001883446, "naucs_at_100_std": 0.32571156994244094, "naucs_at_100_diff1": -0.11435432105284754 }, "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { "ndcg_at_1": 0.67241, "ndcg_at_3": 0.63319, "ndcg_at_5": 0.58758, "ndcg_at_10": 0.57198, "ndcg_at_20": 0.58918, "ndcg_at_50": 0.65786, "ndcg_at_100": 0.69503, "map_at_1": 0.13615, "map_at_3": 0.21649, "map_at_5": 0.25675, "map_at_10": 0.3252, "map_at_20": 0.38016, "map_at_50": 0.43771, "map_at_100": 0.46413, "recall_at_1": 0.13615, "recall_at_3": 0.23646, "recall_at_5": 0.29799, "recall_at_10": 0.43273, "recall_at_20": 0.56259, "recall_at_50": 0.76924, "recall_at_100": 0.88035, "precision_at_1": 0.67241, "precision_at_3": 0.57471, "precision_at_5": 0.48966, "precision_at_10": 0.39828, "precision_at_20": 0.29914, "precision_at_50": 0.19172, "precision_at_100": 0.12379, "mrr_at_1": 0.6724137931034483, "mrr_at_3": 0.7557471264367817, "mrr_at_5": 0.764367816091954, "mrr_at_10": 0.7718596059113301, "mrr_at_20": 0.7718596059113301, "mrr_at_50": 0.7732378352919377, "mrr_at_100": 0.7732378352919377, "naucs_at_1_max": 0.032282867498218674, "naucs_at_1_std": -0.28271414000179634, "naucs_at_1_diff1": 0.14417521370616468, "naucs_at_3_max": 0.19868656246441452, "naucs_at_3_std": 0.12756096614672727, "naucs_at_3_diff1": -0.14423513218105685, "naucs_at_5_max": 0.16258116633927042, "naucs_at_5_std": 0.08888872261627004, "naucs_at_5_diff1": -0.09215214837175129, "naucs_at_10_max": 0.08087197701067339, "naucs_at_10_std": 0.0593535492140523, "naucs_at_10_diff1": 0.011413698146931404, "naucs_at_20_max": 0.09066512825980459, "naucs_at_20_std": 0.1515192296500721, "naucs_at_20_diff1": 0.05635159011980896, "naucs_at_50_max": 0.07419981716098847, "naucs_at_50_std": 0.18498184475514726, "naucs_at_50_diff1": 0.041495490587423654, "naucs_at_100_max": 0.03526540983660958, "naucs_at_100_std": 0.11166203694040092, "naucs_at_100_diff1": 0.12019253855015599 }, "vidore/synthetic_rse_restaurant_filtered_v1.0": { "ndcg_at_1": 0.42105, "ndcg_at_3": 0.446, "ndcg_at_5": 0.4719, "ndcg_at_10": 0.51181, "ndcg_at_20": 0.5424, "ndcg_at_50": 0.56432, "ndcg_at_100": 0.5872, "map_at_1": 0.18615, "map_at_3": 0.3094, "map_at_5": 0.34722, "map_at_10": 0.38335, "map_at_20": 0.40112, "map_at_50": 0.40968, "map_at_100": 0.41626, "recall_at_1": 0.18615, "recall_at_3": 0.42928, "recall_at_5": 0.52991, "recall_at_10": 0.66591, "recall_at_20": 0.76433, "recall_at_50": 0.83751, "recall_at_100": 0.91575, "precision_at_1": 0.42105, "precision_at_3": 0.30409, "precision_at_5": 0.24561, "precision_at_10": 0.16667, "precision_at_20": 0.10789, "precision_at_50": 0.05333, "precision_at_100": 0.03263, "mrr_at_1": 0.42105263157894735, "mrr_at_3": 0.5350877192982455, "mrr_at_5": 0.5552631578947367, "mrr_at_10": 0.5674463937621831, "mrr_at_20": 0.5723974683727004, "mrr_at_50": 0.5730024290502564, "mrr_at_100": 0.5730024290502564, "naucs_at_1_max": -0.1063878813786229, "naucs_at_1_std": 0.07009527096294454, "naucs_at_1_diff1": 0.10532570058239055, "naucs_at_3_max": -0.0707818319042163, "naucs_at_3_std": 0.10106279988011081, "naucs_at_3_diff1": 0.084845655655151, "naucs_at_5_max": -0.11487901190848457, "naucs_at_5_std": 0.09251915109792752, "naucs_at_5_diff1": 0.012535607370615548, "naucs_at_10_max": -0.09127037923640684, "naucs_at_10_std": 0.13093656784846772, "naucs_at_10_diff1": 0.03006202135754748, "naucs_at_20_max": -0.05693043074070981, "naucs_at_20_std": 0.1422048179530491, "naucs_at_20_diff1": 0.021927827197454224, "naucs_at_50_max": -0.12190658822908639, "naucs_at_50_std": 0.16430064525199148, "naucs_at_50_diff1": -0.10417541492717723, "naucs_at_100_max": -0.17484165527500634, "naucs_at_100_std": 0.13672197776470205, "naucs_at_100_diff1": -0.15839101235075956 }, "vidore/synthetic_axa_filtered_v1.0": { "ndcg_at_1": 0.72222, "ndcg_at_3": 0.66954, "ndcg_at_5": 0.64088, "ndcg_at_10": 0.6477, "ndcg_at_20": 0.68679, "ndcg_at_50": 0.73939, "ndcg_at_100": 0.75782, "map_at_1": 0.36677, "map_at_3": 0.4592, "map_at_5": 0.48529, "map_at_10": 0.53077, "map_at_20": 0.56672, "map_at_50": 0.58988, "map_at_100": 0.5958, "recall_at_1": 0.36677, "recall_at_3": 0.50423, "recall_at_5": 0.5832, "recall_at_10": 0.66079, "recall_at_20": 0.77947, "recall_at_50": 0.9373, "recall_at_100": 0.99068, "precision_at_1": 0.72222, "precision_at_3": 0.44444, "precision_at_5": 0.32222, "precision_at_10": 0.23333, "precision_at_20": 0.15833, "precision_at_50": 0.08333, "precision_at_100": 0.04667, "mrr_at_1": 0.7222222222222222, "mrr_at_3": 0.7592592592592593, "mrr_at_5": 0.7842592592592593, "mrr_at_10": 0.7842592592592593, "mrr_at_20": 0.7842592592592593, "mrr_at_50": 0.7856481481481482, "mrr_at_100": 0.7856481481481482, "naucs_at_1_max": -0.5764508261912696, "naucs_at_1_std": 0.2572205519822324, "naucs_at_1_diff1": 0.5945012986070088, "naucs_at_3_max": -0.3245546955664279, "naucs_at_3_std": 0.2106608705520064, "naucs_at_3_diff1": -0.18354778613695244, "naucs_at_5_max": -0.32979775271957523, "naucs_at_5_std": 0.004295616052457978, "naucs_at_5_diff1": -0.21668629588661706, "naucs_at_10_max": -0.37966732415422166, "naucs_at_10_std": -0.01491020124585064, "naucs_at_10_diff1": -0.17349036538288762, "naucs_at_20_max": -0.29338316737381204, "naucs_at_20_std": -0.03576033793090156, "naucs_at_20_diff1": -0.19533219218843523, "naucs_at_50_max": -0.29990699257368786, "naucs_at_50_std": -0.11316959031750934, "naucs_at_50_diff1": -0.27924927011864364, "naucs_at_100_max": -0.22547374701018, "naucs_at_100_std": -0.1279133290410171, "naucs_at_100_diff1": -0.32901479145609697 }, "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { "ndcg_at_1": 0.41667, "ndcg_at_3": 0.44206, "ndcg_at_5": 0.46171, "ndcg_at_10": 0.50911, "ndcg_at_20": 0.54735, "ndcg_at_50": 0.5785, "ndcg_at_100": 0.59513, "map_at_1": 0.18884, "map_at_3": 0.30884, "map_at_5": 0.34205, "map_at_10": 0.38214, "map_at_20": 0.40339, "map_at_50": 0.41645, "map_at_100": 0.42174, "recall_at_1": 0.18884, "recall_at_3": 0.41439, "recall_at_5": 0.50081, "recall_at_10": 0.65362, "recall_at_20": 0.77708, "recall_at_50": 0.87498, "recall_at_100": 0.92944, "precision_at_1": 0.41667, "precision_at_3": 0.30409, "precision_at_5": 0.2386, "precision_at_10": 0.16667, "precision_at_20": 0.10965, "precision_at_50": 0.05763, "precision_at_100": 0.03364, "mrr_at_1": 0.4166666666666667, "mrr_at_3": 0.5387426900584793, "mrr_at_5": 0.5527777777777776, "mrr_at_10": 0.5685202589807851, "mrr_at_20": 0.5744233342337056, "mrr_at_50": 0.574930578048572, "mrr_at_100": 0.574930578048572, "naucs_at_1_max": 0.01715145223542086, "naucs_at_1_std": 0.0899310510434977, "naucs_at_1_diff1": 0.1739307636238112, "naucs_at_3_max": -0.031778970284482205, "naucs_at_3_std": 0.15113736191861826, "naucs_at_3_diff1": -0.023575716122440124, "naucs_at_5_max": -0.11291764491834187, "naucs_at_5_std": 0.10256509751947128, "naucs_at_5_diff1": -0.07511697723029634, "naucs_at_10_max": -0.06154590672598315, "naucs_at_10_std": 0.08064531970193034, "naucs_at_10_diff1": -0.018188695451970553, "naucs_at_20_max": -0.014776270960712531, "naucs_at_20_std": 0.17295501151323417, "naucs_at_20_diff1": -0.06161735463033105, "naucs_at_50_max": -0.05048590733744633, "naucs_at_50_std": 0.1490648093736084, "naucs_at_50_diff1": -0.13782527489285434, "naucs_at_100_max": -0.06129053077443416, "naucs_at_100_std": 0.1378046540638781, "naucs_at_100_diff1": -0.16944450675960782 }, "vidore/synthetic_axa_filtered_v1.0_multilingual": { "ndcg_at_1": 0.65278, "ndcg_at_3": 0.61596, "ndcg_at_5": 0.59512, "ndcg_at_10": 0.60056, "ndcg_at_20": 0.63495, "ndcg_at_50": 0.69554, "ndcg_at_100": 0.71726, "map_at_1": 0.33339, "map_at_3": 0.4226, "map_at_5": 0.45105, "map_at_10": 0.49113, "map_at_20": 0.52543, "map_at_50": 0.5505, "map_at_100": 0.55603, "recall_at_1": 0.33339, "recall_at_3": 0.46525, "recall_at_5": 0.5196, "recall_at_10": 0.60514, "recall_at_20": 0.70862, "recall_at_50": 0.89775, "recall_at_100": 0.97606, "precision_at_1": 0.65278, "precision_at_3": 0.40741, "precision_at_5": 0.31667, "precision_at_10": 0.22361, "precision_at_20": 0.14861, "precision_at_50": 0.08111, "precision_at_100": 0.04528, "mrr_at_1": 0.6527777777777778, "mrr_at_3": 0.7037037037037037, "mrr_at_5": 0.7203703703703704, "mrr_at_10": 0.7238977072310406, "mrr_at_20": 0.7247657627865962, "mrr_at_50": 0.7270465535079976, "mrr_at_100": 0.7274945821818327, "naucs_at_1_max": 0.22507029768221098, "naucs_at_1_std": 0.13788316928206162, "naucs_at_1_diff1": 0.5995848183367006, "naucs_at_3_max": 0.10374741818519737, "naucs_at_3_std": 0.1212402799460214, "naucs_at_3_diff1": 0.032982270408976666, "naucs_at_5_max": 0.036545011697732825, "naucs_at_5_std": 0.04976840467312129, "naucs_at_5_diff1": -0.06474423852856963, "naucs_at_10_max": -0.012180820225981331, "naucs_at_10_std": 0.007819844618087223, "naucs_at_10_diff1": -0.11570751534998236, "naucs_at_20_max": -0.05746500182397195, "naucs_at_20_std": -0.02111513643125022, "naucs_at_20_diff1": -0.12602864520641527, "naucs_at_50_max": -0.1124769198431645, "naucs_at_50_std": -0.07628706085741649, "naucs_at_50_diff1": -0.19607255004920401, "naucs_at_100_max": -0.11390067714346258, "naucs_at_100_std": -0.09427043796285493, "naucs_at_100_diff1": -0.25292751670835045 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { "ndcg_at_1": 0.46875, "ndcg_at_3": 0.48885, "ndcg_at_5": 0.51496, "ndcg_at_10": 0.54548, "ndcg_at_20": 0.57449, "ndcg_at_50": 0.60556, "ndcg_at_100": 0.62078, "map_at_1": 0.27801, "map_at_3": 0.38689, "map_at_5": 0.42623, "map_at_10": 0.45364, "map_at_20": 0.46929, "map_at_50": 0.47926, "map_at_100": 0.48215, "recall_at_1": 0.27801, "recall_at_3": 0.46662, "recall_at_5": 0.55996, "recall_at_10": 0.65464, "recall_at_20": 0.73468, "recall_at_50": 0.83743, "recall_at_100": 0.90162, "precision_at_1": 0.46875, "precision_at_3": 0.30781, "precision_at_5": 0.24, "precision_at_10": 0.15109, "precision_at_20": 0.09297, "precision_at_50": 0.04666, "precision_at_100": 0.02609, "mrr_at_1": 0.46875, "mrr_at_3": 0.5653645833333331, "mrr_at_5": 0.5828645833333328, "mrr_at_10": 0.5928670634920632, "mrr_at_20": 0.5961635393061956, "mrr_at_50": 0.5979642273664314, "mrr_at_100": 0.598388641842729, "naucs_at_1_max": 0.25717698422505025, "naucs_at_1_std": -0.10478191029284567, "naucs_at_1_diff1": 0.4072573005477985, "naucs_at_3_max": 0.2659990641306254, "naucs_at_3_std": 0.04479561155056106, "naucs_at_3_diff1": 0.11139438643390147, "naucs_at_5_max": 0.2170152015957716, "naucs_at_5_std": 0.06816504281074703, "naucs_at_5_diff1": 0.03610018938984803, "naucs_at_10_max": 0.23222160900815464, "naucs_at_10_std": 0.1310209006859444, "naucs_at_10_diff1": -0.038456197207725154, "naucs_at_20_max": 0.27497446589778796, "naucs_at_20_std": 0.208324896139066, "naucs_at_20_diff1": -0.08762814336480476, "naucs_at_50_max": 0.27441764448304734, "naucs_at_50_std": 0.2529300579928729, "naucs_at_50_diff1": -0.11931330945117197, "naucs_at_100_max": 0.26643076503646324, "naucs_at_100_std": 0.23283573973890204, "naucs_at_100_diff1": -0.09996794144423607 }, "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { "ndcg_at_1": 0.64224, "ndcg_at_3": 0.60064, "ndcg_at_5": 0.55019, "ndcg_at_10": 0.52755, "ndcg_at_20": 0.55431, "ndcg_at_50": 0.62193, "ndcg_at_100": 0.66273, "map_at_1": 0.12328, "map_at_3": 0.20176, "map_at_5": 0.23781, "map_at_10": 0.29317, "map_at_20": 0.3496, "map_at_50": 0.40329, "map_at_100": 0.42827, "recall_at_1": 0.12328, "recall_at_3": 0.22334, "recall_at_5": 0.27732, "recall_at_10": 0.39122, "recall_at_20": 0.54233, "recall_at_50": 0.74043, "recall_at_100": 0.86408, "precision_at_1": 0.64224, "precision_at_3": 0.54023, "precision_at_5": 0.45345, "precision_at_10": 0.36509, "precision_at_20": 0.28384, "precision_at_50": 0.18336, "precision_at_100": 0.11944, "mrr_at_1": 0.6422413793103449, "mrr_at_3": 0.726293103448276, "mrr_at_5": 0.7344827586206897, "mrr_at_10": 0.7423662424740011, "mrr_at_20": 0.7454521368427901, "mrr_at_50": 0.7464683101612821, "mrr_at_100": 0.7466140292051692, "naucs_at_1_max": 0.04669677333287125, "naucs_at_1_std": -0.047276436301337176, "naucs_at_1_diff1": 0.20998438017074644, "naucs_at_3_max": 0.1207746945432274, "naucs_at_3_std": 0.11705226278367024, "naucs_at_3_diff1": -0.0659769870275157, "naucs_at_5_max": 0.17489805487159318, "naucs_at_5_std": 0.15087482671660668, "naucs_at_5_diff1": -0.10486886234938332, "naucs_at_10_max": 0.13971495080647375, "naucs_at_10_std": 0.12342403085916763, "naucs_at_10_diff1": -0.11904046578732998, "naucs_at_20_max": 0.1262352504070948, "naucs_at_20_std": 0.1870795410571109, "naucs_at_20_diff1": -0.09021505632729916, "naucs_at_50_max": 0.08609145496415921, "naucs_at_50_std": 0.19880949579195997, "naucs_at_50_diff1": -0.10141550865230196, "naucs_at_100_max": 0.029463316752851777, "naucs_at_100_std": 0.1254397237152911, "naucs_at_100_diff1": -0.10340073450123488 }, "vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.78, "ndcg_at_3": 0.83717, "ndcg_at_5": 0.8493, "ndcg_at_10": 0.85579, "ndcg_at_20": 0.86416, "ndcg_at_50": 0.86978, "ndcg_at_100": 0.8714, "map_at_1": 0.78, "map_at_3": 0.82367, "map_at_5": 0.83027, "map_at_10": 0.83296, "map_at_20": 0.83513, "map_at_50": 0.83606, "map_at_100": 0.8362, "recall_at_1": 0.78, "recall_at_3": 0.876, "recall_at_5": 0.906, "recall_at_10": 0.926, "recall_at_20": 0.96, "recall_at_50": 0.988, "recall_at_100": 0.998, "precision_at_1": 0.78, "precision_at_3": 0.292, "precision_at_5": 0.1812, "precision_at_10": 0.0926, "precision_at_20": 0.048, "precision_at_50": 0.01976, "precision_at_100": 0.00998, "mrr_at_1": 0.78, "mrr_at_3": 0.8236666666666668, "mrr_at_5": 0.8302666666666666, "mrr_at_10": 0.8329571428571427, "mrr_at_20": 0.8351287569413728, "mrr_at_50": 0.8360576689817807, "mrr_at_100": 0.8362028700500819, "naucs_at_1_max": 0.7913757128946993, "naucs_at_1_std": -0.6034404414151248, "naucs_at_1_diff1": 0.9052951268141137, "naucs_at_3_max": 0.7991254905596246, "naucs_at_3_std": -0.5793856802941784, "naucs_at_3_diff1": 0.8585874972961292, "naucs_at_5_max": 0.7909291376124926, "naucs_at_5_std": -0.4963148379919354, "naucs_at_5_diff1": 0.8356576673222494, "naucs_at_10_max": 0.8344689227042192, "naucs_at_10_std": -0.5341433870845648, "naucs_at_10_diff1": 0.8497867615514662, "naucs_at_20_max": 0.8126984126984111, "naucs_at_20_std": -0.4176003734827309, "naucs_at_20_diff1": 0.8263305322128826, "naucs_at_50_max": 0.9074074074074128, "naucs_at_50_std": -0.04474011826952445, "naucs_at_50_diff1": 0.860333022097726, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.45455, "ndcg_at_3": 0.53012, "ndcg_at_5": 0.55569, "ndcg_at_10": 0.57792, "ndcg_at_20": 0.59293, "ndcg_at_50": 0.60814, "ndcg_at_100": 0.61994, "map_at_1": 0.45455, "map_at_3": 0.51109, "map_at_5": 0.52528, "map_at_10": 0.53447, "map_at_20": 0.53882, "map_at_50": 0.54136, "map_at_100": 0.54249, "recall_at_1": 0.45455, "recall_at_3": 0.58537, "recall_at_5": 0.64745, "recall_at_10": 0.71619, "recall_at_20": 0.77384, "recall_at_50": 0.84922, "recall_at_100": 0.92018, "precision_at_1": 0.45455, "precision_at_3": 0.19512, "precision_at_5": 0.12949, "precision_at_10": 0.07162, "precision_at_20": 0.03869, "precision_at_50": 0.01698, "precision_at_100": 0.0092, "mrr_at_1": 0.45454545454545453, "mrr_at_3": 0.5110864745011089, "mrr_at_5": 0.5252771618625277, "mrr_at_10": 0.5344683771513038, "mrr_at_20": 0.5388215531769072, "mrr_at_50": 0.5413611772588918, "mrr_at_100": 0.5424869155538622, "naucs_at_1_max": 0.4075821857009155, "naucs_at_1_std": -0.310949225760234, "naucs_at_1_diff1": 0.7407514800732501, "naucs_at_3_max": 0.46743175097720985, "naucs_at_3_std": -0.1986126839848541, "naucs_at_3_diff1": 0.6505296132803555, "naucs_at_5_max": 0.5364564428320254, "naucs_at_5_std": -0.14408657067489153, "naucs_at_5_diff1": 0.6322410022842855, "naucs_at_10_max": 0.5825512151708625, "naucs_at_10_std": -0.07725749814895451, "naucs_at_10_diff1": 0.6332882465209557, "naucs_at_20_max": 0.5813970577815284, "naucs_at_20_std": -0.02416200306109636, "naucs_at_20_diff1": 0.5952214798946482, "naucs_at_50_max": 0.6298047922819098, "naucs_at_50_std": 0.21852001263213766, "naucs_at_50_diff1": 0.5810079368486553, "naucs_at_100_max": 0.6564226194686283, "naucs_at_100_std": 0.46804642848156325, "naucs_at_100_diff1": 0.49830945198923904 }, "vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.7753, "ndcg_at_3": 0.84285, "ndcg_at_5": 0.85383, "ndcg_at_10": 0.86362, "ndcg_at_20": 0.86829, "ndcg_at_50": 0.87162, "ndcg_at_100": 0.87297, "map_at_1": 0.7753, "map_at_3": 0.82692, "map_at_5": 0.8331, "map_at_10": 0.83712, "map_at_20": 0.83843, "map_at_50": 0.83902, "map_at_100": 0.83915, "recall_at_1": 0.7753, "recall_at_3": 0.88866, "recall_at_5": 0.91498, "recall_at_10": 0.94534, "recall_at_20": 0.96356, "recall_at_50": 0.97976, "recall_at_100": 0.98785, "precision_at_1": 0.7753, "precision_at_3": 0.29622, "precision_at_5": 0.183, "precision_at_10": 0.09453, "precision_at_20": 0.04818, "precision_at_50": 0.0196, "precision_at_100": 0.00988, "mrr_at_1": 0.7753036437246964, "mrr_at_3": 0.8269230769230768, "mrr_at_5": 0.8330971659919026, "mrr_at_10": 0.8371192404087138, "mrr_at_20": 0.8384322229490562, "mrr_at_50": 0.8390209545984961, "mrr_at_100": 0.8391494570651827, "naucs_at_1_max": 0.42955277741228204, "naucs_at_1_std": -0.413540459458001, "naucs_at_1_diff1": 0.8615855669987156, "naucs_at_3_max": 0.46912857131917574, "naucs_at_3_std": -0.3714926257299717, "naucs_at_3_diff1": 0.82680032161878, "naucs_at_5_max": 0.6392482503231406, "naucs_at_5_std": -0.2299842899342104, "naucs_at_5_diff1": 0.8417184248940351, "naucs_at_10_max": 0.7086238741706962, "naucs_at_10_std": -0.0498141562896836, "naucs_at_10_diff1": 0.837619456912457, "naucs_at_20_max": 0.6570477401161009, "naucs_at_20_std": 0.0009204781615544056, "naucs_at_20_diff1": 0.7936258020327029, "naucs_at_50_max": 0.774779293140126, "naucs_at_50_std": 0.35856346999101907, "naucs_at_50_diff1": 0.7547641636196891, "naucs_at_100_max": 0.7116997912767351, "naucs_at_100_std": 0.3194376069347773, "naucs_at_100_diff1": 0.7833756960419608 }, "vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.87143, "ndcg_at_3": 0.91866, "ndcg_at_5": 0.92758, "ndcg_at_10": 0.92996, "ndcg_at_20": 0.9336, "ndcg_at_50": 0.93499, "ndcg_at_100": 0.93499, "map_at_1": 0.87143, "map_at_3": 0.90774, "map_at_5": 0.91274, "map_at_10": 0.91376, "map_at_20": 0.91476, "map_at_50": 0.91497, "map_at_100": 0.91497, "recall_at_1": 0.87143, "recall_at_3": 0.95, "recall_at_5": 0.97143, "recall_at_10": 0.97857, "recall_at_20": 0.99286, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.87143, "precision_at_3": 0.31667, "precision_at_5": 0.19429, "precision_at_10": 0.09786, "precision_at_20": 0.04964, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8714285714285714, "mrr_at_3": 0.9077380952380953, "mrr_at_5": 0.9127380952380953, "mrr_at_10": 0.9137585034013606, "mrr_at_20": 0.9147645211930926, "mrr_at_50": 0.9149716040262259, "mrr_at_100": 0.9149716040262259, "naucs_at_1_max": 0.7871300352601777, "naucs_at_1_std": 0.20583128539373943, "naucs_at_1_diff1": 0.9197964526124592, "naucs_at_3_max": 0.9229691876750705, "naucs_at_3_std": 0.460784313725494, "naucs_at_3_diff1": 0.9626517273576086, "naucs_at_5_max": 0.9673202614379075, "naucs_at_5_std": 0.562791783380022, "naucs_at_5_diff1": 0.934640522875815, "naucs_at_10_max": 0.9782135076252753, "naucs_at_10_std": 0.7584811702458847, "naucs_at_10_diff1": 0.9128540305010935, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 0.8692810457516309, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/tatdqa_test": { "ndcg_at_1": 0.55346, "ndcg_at_3": 0.66654, "ndcg_at_5": 0.69922, "ndcg_at_10": 0.72683, "ndcg_at_20": 0.73868, "ndcg_at_50": 0.74458, "ndcg_at_100": 0.74555, "map_at_1": 0.55346, "map_at_3": 0.63902, "map_at_5": 0.65725, "map_at_10": 0.66893, "map_at_20": 0.67236, "map_at_50": 0.6733, "map_at_100": 0.67338, "recall_at_1": 0.55346, "recall_at_3": 0.74605, "recall_at_5": 0.82503, "recall_at_10": 0.90887, "recall_at_20": 0.95443, "recall_at_50": 0.9842, "recall_at_100": 0.99028, "precision_at_1": 0.55346, "precision_at_3": 0.24868, "precision_at_5": 0.16501, "precision_at_10": 0.09089, "precision_at_20": 0.04772, "precision_at_50": 0.01968, "precision_at_100": 0.0099, "mrr_at_1": 0.5552855407047388, "mrr_at_3": 0.6395301741595798, "mrr_at_5": 0.6578473066018649, "mrr_at_10": 0.6696035603386765, "mrr_at_20": 0.6729832012543134, "mrr_at_50": 0.6739181612518058, "mrr_at_100": 0.6739995114642742, "naucs_at_1_max": 0.4481578507912886, "naucs_at_1_std": -0.28229697462188064, "naucs_at_1_diff1": 0.6794797414689293, "naucs_at_3_max": 0.521452354349859, "naucs_at_3_std": -0.2525394663950513, "naucs_at_3_diff1": 0.5641368811967125, "naucs_at_5_max": 0.5522320601024093, "naucs_at_5_std": -0.23449944203419384, "naucs_at_5_diff1": 0.5550996108687463, "naucs_at_10_max": 0.6709947806703614, "naucs_at_10_std": -0.018864423203824147, "naucs_at_10_diff1": 0.5147848561655853, "naucs_at_20_max": 0.6719932800208073, "naucs_at_20_std": 0.18814578113635355, "naucs_at_20_diff1": 0.4774949408808388, "naucs_at_50_max": 0.710665729775863, "naucs_at_50_std": 0.409805723201301, "naucs_at_50_diff1": 0.4920058027972864, "naucs_at_100_max": 0.6822698211447887, "naucs_at_100_std": 0.3946630393669552, "naucs_at_100_diff1": 0.5232939215033167 }, "vidore/shiftproject_test": { "ndcg_at_1": 0.59, "ndcg_at_3": 0.76142, "ndcg_at_5": 0.78683, "ndcg_at_10": 0.8064, "ndcg_at_20": 0.8064, "ndcg_at_50": 0.8064, "ndcg_at_100": 0.80812, "map_at_1": 0.59, "map_at_3": 0.72333, "map_at_5": 0.73783, "map_at_10": 0.74602, "map_at_20": 0.74602, "map_at_50": 0.74602, "map_at_100": 0.74621, "recall_at_1": 0.59, "recall_at_3": 0.87, "recall_at_5": 0.93, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.59, "precision_at_3": 0.29, "precision_at_5": 0.186, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.59, "mrr_at_3": 0.7233333333333333, "mrr_at_5": 0.7378333333333333, "mrr_at_10": 0.7460238095238095, "mrr_at_20": 0.7460238095238095, "mrr_at_50": 0.7460238095238095, "mrr_at_100": 0.7462056277056278, "naucs_at_1_max": 0.14863456938676678, "naucs_at_1_std": -0.41319500491584377, "naucs_at_1_diff1": 0.6677623373002481, "naucs_at_3_max": 0.34644153151151447, "naucs_at_3_std": -0.3760275494334602, "naucs_at_3_diff1": 0.5028141894393838, "naucs_at_5_max": 0.29825263438709065, "naucs_at_5_std": -0.40456182472989133, "naucs_at_5_diff1": 0.434507136187809, "naucs_at_10_max": -0.1713352007469681, "naucs_at_10_std": -0.5634920634920767, "naucs_at_10_diff1": -1.1517273576097316, "naucs_at_20_max": -0.1713352007469681, "naucs_at_20_std": -0.5634920634920767, "naucs_at_20_diff1": -1.1517273576097316, "naucs_at_50_max": -0.17133520074697067, "naucs_at_50_std": -0.5634920634920583, "naucs_at_50_diff1": -1.1517273576096694, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.87, "ndcg_at_3": 0.94047, "ndcg_at_5": 0.94478, "ndcg_at_10": 0.94478, "ndcg_at_20": 0.94478, "ndcg_at_50": 0.94478, "ndcg_at_100": 0.94478, "map_at_1": 0.87, "map_at_3": 0.92333, "map_at_5": 0.92583, "map_at_10": 0.92583, "map_at_20": 0.92583, "map_at_50": 0.92583, "map_at_100": 0.92583, "recall_at_1": 0.87, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.87, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.87, "mrr_at_3": 0.9233333333333333, "mrr_at_5": 0.9258333333333333, "mrr_at_10": 0.9258333333333333, "mrr_at_20": 0.9258333333333333, "mrr_at_50": 0.9258333333333333, "mrr_at_100": 0.9258333333333333, "naucs_at_1_max": 0.37958231504110235, "naucs_at_1_std": 0.006480041472265402, "naucs_at_1_diff1": 0.9063171147152481, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.7222222222222157, "naucs_at_3_diff1": 0.8692810457516356, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.85, "ndcg_at_3": 0.89155, "ndcg_at_5": 0.89542, "ndcg_at_10": 0.91169, "ndcg_at_20": 0.91448, "ndcg_at_50": 0.91661, "ndcg_at_100": 0.91661, "map_at_1": 0.85, "map_at_3": 0.88167, "map_at_5": 0.88367, "map_at_10": 0.89044, "map_at_20": 0.89135, "map_at_50": 0.89175, "map_at_100": 0.89175, "recall_at_1": 0.85, "recall_at_3": 0.92, "recall_at_5": 0.93, "recall_at_10": 0.98, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.85, "precision_at_3": 0.30667, "precision_at_5": 0.186, "precision_at_10": 0.098, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.85, "mrr_at_3": 0.8816666666666667, "mrr_at_5": 0.8836666666666667, "mrr_at_10": 0.8904404761904763, "mrr_at_20": 0.8913495670995673, "mrr_at_50": 0.8917495670995673, "mrr_at_100": 0.8917495670995673, "naucs_at_1_max": 0.36173120728929387, "naucs_at_1_std": -0.9650178978197211, "naucs_at_1_diff1": 0.9199479336153596, "naucs_at_3_max": 0.4982492997198882, "naucs_at_3_std": -1.0291783380018664, "naucs_at_3_diff1": 0.932598039215688, "naucs_at_5_max": 0.5939042283580102, "naucs_at_5_std": -1.0116713352007418, "naucs_at_5_diff1": 0.96265172735761, "naucs_at_10_max": 0.21825396825397442, "naucs_at_10_std": -1.7399626517273414, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": -1.7399626517273863, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.9, "ndcg_at_3": 0.96047, "ndcg_at_5": 0.96047, "ndcg_at_10": 0.96047, "ndcg_at_20": 0.96047, "ndcg_at_50": 0.96047, "ndcg_at_100": 0.96047, "map_at_1": 0.9, "map_at_3": 0.94667, "map_at_5": 0.94667, "map_at_10": 0.94667, "map_at_20": 0.94667, "map_at_50": 0.94667, "map_at_100": 0.94667, "recall_at_1": 0.9, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9, "mrr_at_3": 0.9466666666666668, "mrr_at_5": 0.9466666666666668, "mrr_at_10": 0.9466666666666668, "mrr_at_20": 0.9466666666666668, "mrr_at_50": 0.9466666666666668, "mrr_at_100": 0.9466666666666668, "naucs_at_1_max": 0.2663398692810457, "naucs_at_1_std": -0.1846405228758159, "naucs_at_1_diff1": 0.8835200746965451, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.87, "ndcg_at_3": 0.95071, "ndcg_at_5": 0.95071, "ndcg_at_10": 0.95071, "ndcg_at_20": 0.95071, "ndcg_at_50": 0.95071, "ndcg_at_100": 0.95071, "map_at_1": 0.87, "map_at_3": 0.93333, "map_at_5": 0.93333, "map_at_10": 0.93333, "map_at_20": 0.93333, "map_at_50": 0.93333, "map_at_100": 0.93333, "recall_at_1": 0.87, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.87, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.87, "mrr_at_3": 0.9333333333333332, "mrr_at_5": 0.9333333333333332, "mrr_at_10": 0.9333333333333332, "mrr_at_20": 0.9333333333333332, "mrr_at_50": 0.9333333333333332, "mrr_at_100": 0.9333333333333332, "naucs_at_1_max": 0.640339183885063, "naucs_at_1_std": -0.4568984670073326, "naucs_at_1_diff1": 0.9339035769828927, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null } } }