{ "metadata": { "timestamp": "2025-04-16T15:52:00.834269", "vidore_benchmark_version": "5.0.1.dev6+g9e0da63" }, "metrics": { "./data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.97, "ndcg_at_3": 0.98262, "ndcg_at_5": 0.98693, "ndcg_at_10": 0.98693, "ndcg_at_20": 0.98693, "ndcg_at_50": 0.98693, "ndcg_at_100": 0.98693, "map_at_1": 0.97, "map_at_3": 0.98, "map_at_5": 0.9825, "map_at_10": 0.9825, "map_at_20": 0.9825, "map_at_50": 0.9825, "map_at_100": 0.9825, "recall_at_1": 0.97, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.97, "mrr_at_3": 0.98, "mrr_at_5": 0.9825, "mrr_at_10": 0.9825, "mrr_at_20": 0.9825, "mrr_at_50": 0.9825, "mrr_at_100": 0.9825, "naucs_at_1_max": 0.7642390289449127, "naucs_at_1_std": -0.02754435107376295, "naucs_at_1_diff1": 0.9564270152505465, "naucs_at_3_max": 0.8692810457516356, "naucs_at_3_std": -0.5634920634921204, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "./data_dir/eval_vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.91, "ndcg_at_3": 0.94155, "ndcg_at_5": 0.94585, "ndcg_at_10": 0.94585, "ndcg_at_20": 0.95135, "ndcg_at_50": 0.9535, "ndcg_at_100": 0.9535, "map_at_1": 0.91, "map_at_3": 0.935, "map_at_5": 0.9375, "map_at_10": 0.9375, "map_at_20": 0.93924, "map_at_50": 0.93966, "map_at_100": 0.93966, "recall_at_1": 0.91, "recall_at_3": 0.96, "recall_at_5": 0.97, "recall_at_10": 0.97, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.91, "precision_at_3": 0.32, "precision_at_5": 0.194, "precision_at_10": 0.097, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.91, "mrr_at_3": 0.935, "mrr_at_5": 0.9375, "mrr_at_10": 0.9384999999999999, "mrr_at_20": 0.9393333333333334, "mrr_at_50": 0.93975, "mrr_at_100": 0.93975, "naucs_at_1_max": 0.5892208735345978, "naucs_at_1_std": -0.26906318082788605, "naucs_at_1_diff1": 0.9709513435003629, "naucs_at_3_max": 0.73739495798319, "naucs_at_3_std": -0.7742763772175512, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.6934329287270459, "naucs_at_5_std": -1.2170868347338937, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.6934329287270459, "naucs_at_10_std": -1.2170868347338937, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": -1.7399626517273863, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "./data_dir/eval_vidore/tatdqa_test": { "ndcg_at_1": 0.71081, "ndcg_at_3": 0.80334, "ndcg_at_5": 0.82102, "ndcg_at_10": 0.83392, "ndcg_at_20": 0.83983, "ndcg_at_50": 0.84192, "ndcg_at_100": 0.8433, "map_at_1": 0.71081, "map_at_3": 0.78109, "map_at_5": 0.79099, "map_at_10": 0.79628, "map_at_20": 0.79794, "map_at_50": 0.79829, "map_at_100": 0.79841, "recall_at_1": 0.71081, "recall_at_3": 0.86756, "recall_at_5": 0.91009, "recall_at_10": 0.95018, "recall_at_20": 0.97327, "recall_at_50": 0.9836, "recall_at_100": 0.9921, "precision_at_1": 0.71081, "precision_at_3": 0.28919, "precision_at_5": 0.18202, "precision_at_10": 0.09502, "precision_at_20": 0.04866, "precision_at_50": 0.01967, "precision_at_100": 0.00992, "mrr_at_1": 0.7102065613608749, "mrr_at_3": 0.7792628594572701, "mrr_at_5": 0.7895605508302956, "mrr_at_10": 0.7952395417462247, "mrr_at_20": 0.7967528579011667, "mrr_at_50": 0.7971238706885642, "mrr_at_100": 0.7972458544887197, "naucs_at_1_max": 0.27306975389001265, "naucs_at_1_std": -0.2360361174333752, "naucs_at_1_diff1": 0.8141731336960643, "naucs_at_3_max": 0.3002316019256239, "naucs_at_3_std": -0.24363766969519043, "naucs_at_3_diff1": 0.7201261840669434, "naucs_at_5_max": 0.335220118413303, "naucs_at_5_std": -0.1498733056249204, "naucs_at_5_diff1": 0.6849724733759673, "naucs_at_10_max": 0.4358281838742191, "naucs_at_10_std": 0.08882471772211498, "naucs_at_10_diff1": 0.6331592473660967, "naucs_at_20_max": 0.4841367562608157, "naucs_at_20_std": 0.20879545227917629, "naucs_at_20_diff1": 0.5975020128140883, "naucs_at_50_max": 0.5453620297539681, "naucs_at_50_std": 0.3954426713383344, "naucs_at_50_diff1": 0.5292445142004694, "naucs_at_100_max": 0.592298569453559, "naucs_at_100_std": 0.5668305640221949, "naucs_at_100_diff1": 0.6351333165241365 }, "./data_dir/eval_vidore/shiftproject_test": { "ndcg_at_1": 0.76, "ndcg_at_3": 0.86833, "ndcg_at_5": 0.88081, "ndcg_at_10": 0.88415, "ndcg_at_20": 0.88415, "ndcg_at_50": 0.88808, "ndcg_at_100": 0.88808, "map_at_1": 0.76, "map_at_3": 0.84333, "map_at_5": 0.85033, "map_at_10": 0.85176, "map_at_20": 0.85176, "map_at_50": 0.85237, "map_at_100": 0.85237, "recall_at_1": 0.76, "recall_at_3": 0.94, "recall_at_5": 0.97, "recall_at_10": 0.98, "recall_at_20": 0.98, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.76, "precision_at_3": 0.31333, "precision_at_5": 0.194, "precision_at_10": 0.098, "precision_at_20": 0.049, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.76, "mrr_at_3": 0.8466666666666667, "mrr_at_5": 0.8516666666666667, "mrr_at_10": 0.8530952380952381, "mrr_at_20": 0.8530952380952381, "mrr_at_50": 0.8537433862433862, "mrr_at_100": 0.8537433862433862, "naucs_at_1_max": 0.0645063597819505, "naucs_at_1_std": -0.39967119494678566, "naucs_at_1_diff1": 0.741974560872198, "naucs_at_3_max": 0.32197323373794096, "naucs_at_3_std": 0.053299097416744504, "naucs_at_3_diff1": 0.9319172113289744, "naucs_at_5_max": -0.21988795518207335, "naucs_at_5_std": -0.45238095238095277, "naucs_at_5_diff1": 0.9564270152505424, "naucs_at_10_max": -0.5088702147525547, "naucs_at_10_std": -0.5929038281979383, "naucs_at_10_diff1": 0.9346405228758136, "naucs_at_20_max": -0.5088702147525547, "naucs_at_20_std": -0.5929038281979383, "naucs_at_20_diff1": 0.9346405228758136, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "./data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.93, "ndcg_at_3": 0.96655, "ndcg_at_5": 0.97085, "ndcg_at_10": 0.97085, "ndcg_at_20": 0.97085, "ndcg_at_50": 0.97085, "ndcg_at_100": 0.97085, "map_at_1": 0.93, "map_at_3": 0.95833, "map_at_5": 0.96083, "map_at_10": 0.96083, "map_at_20": 0.96083, "map_at_50": 0.96083, "map_at_100": 0.96083, "recall_at_1": 0.93, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.93, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.94, "mrr_at_3": 0.965, "mrr_at_5": 0.9675, "mrr_at_10": 0.9675, "mrr_at_20": 0.9675, "mrr_at_50": 0.9675, "mrr_at_100": 0.9675, "naucs_at_1_max": 0.7885820995064688, "naucs_at_1_std": -0.27757769774576685, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 0.5541549953314585, "naucs_at_3_std": 0.12278244631183229, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "./data_dir/eval_vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.89, "ndcg_at_3": 0.95178, "ndcg_at_5": 0.95565, "ndcg_at_10": 0.95565, "ndcg_at_20": 0.95565, "ndcg_at_50": 0.95565, "ndcg_at_100": 0.95565, "map_at_1": 0.89, "map_at_3": 0.93833, "map_at_5": 0.94033, "map_at_10": 0.94033, "map_at_20": 0.94033, "map_at_50": 0.94033, "map_at_100": 0.94033, "recall_at_1": 0.89, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.89, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.88, "mrr_at_3": 0.9333333333333332, "mrr_at_5": 0.9358333333333333, "mrr_at_10": 0.9358333333333333, "mrr_at_20": 0.9358333333333333, "mrr_at_50": 0.9358333333333333, "mrr_at_100": 0.9358333333333333, "naucs_at_1_max": 0.034275405892964364, "naucs_at_1_std": 0.061120178678808625, "naucs_at_1_diff1": 0.9007817197835238, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.12278244631183229, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "./data_dir/eval_vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.54102, "ndcg_at_3": 0.60463, "ndcg_at_5": 0.6305, "ndcg_at_10": 0.65283, "ndcg_at_20": 0.66498, "ndcg_at_50": 0.68139, "ndcg_at_100": 0.68755, "map_at_1": 0.54102, "map_at_3": 0.58906, "map_at_5": 0.60358, "map_at_10": 0.61285, "map_at_20": 0.61638, "map_at_50": 0.61908, "map_at_100": 0.61963, "recall_at_1": 0.54102, "recall_at_3": 0.64967, "recall_at_5": 0.71175, "recall_at_10": 0.78049, "recall_at_20": 0.82705, "recall_at_50": 0.90909, "recall_at_100": 0.94678, "precision_at_1": 0.54102, "precision_at_3": 0.21656, "precision_at_5": 0.14235, "precision_at_10": 0.07805, "precision_at_20": 0.04135, "precision_at_50": 0.01818, "precision_at_100": 0.00947, "mrr_at_1": 0.5365853658536586, "mrr_at_3": 0.5879526977087953, "mrr_at_5": 0.6025868440502586, "mrr_at_10": 0.6113777144264949, "mrr_at_20": 0.6148145070441927, "mrr_at_50": 0.6173603468552472, "mrr_at_100": 0.617951800317191, "naucs_at_1_max": 0.6730064549090281, "naucs_at_1_std": -0.3132122164452571, "naucs_at_1_diff1": 0.8650865781124909, "naucs_at_3_max": 0.6763294939492751, "naucs_at_3_std": -0.32886290663416623, "naucs_at_3_diff1": 0.7823123375517748, "naucs_at_5_max": 0.6404492018676161, "naucs_at_5_std": -0.3622070751007818, "naucs_at_5_diff1": 0.7366164246094772, "naucs_at_10_max": 0.6637189666157923, "naucs_at_10_std": -0.34234421221627, "naucs_at_10_diff1": 0.7044477964570153, "naucs_at_20_max": 0.6260639953985186, "naucs_at_20_std": -0.3348394344081313, "naucs_at_20_diff1": 0.6800458816048757, "naucs_at_50_max": 0.617806397935773, "naucs_at_50_std": -0.11671744124713705, "naucs_at_50_diff1": 0.6556746075977182, "naucs_at_100_max": 0.5573283569867817, "naucs_at_100_std": -0.09267889165135168, "naucs_at_100_diff1": 0.649669874268164 }, "./data_dir/eval_vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.88462, "ndcg_at_3": 0.91544, "ndcg_at_5": 0.92293, "ndcg_at_10": 0.92907, "ndcg_at_20": 0.93066, "ndcg_at_50": 0.93281, "ndcg_at_100": 0.9341, "map_at_1": 0.88462, "map_at_3": 0.90789, "map_at_5": 0.91204, "map_at_10": 0.91473, "map_at_20": 0.91518, "map_at_50": 0.91559, "map_at_100": 0.9157, "recall_at_1": 0.88462, "recall_at_3": 0.93725, "recall_at_5": 0.95547, "recall_at_10": 0.97368, "recall_at_20": 0.97976, "recall_at_50": 0.98988, "recall_at_100": 0.99798, "precision_at_1": 0.88462, "precision_at_3": 0.31242, "precision_at_5": 0.19109, "precision_at_10": 0.09737, "precision_at_20": 0.04899, "precision_at_50": 0.0198, "precision_at_100": 0.00998, "mrr_at_1": 0.8866396761133604, "mrr_at_3": 0.909919028340081, "mrr_at_5": 0.9134615384615384, "mrr_at_10": 0.916060182507551, "mrr_at_20": 0.9165162042186333, "mrr_at_50": 0.9169019948985417, "mrr_at_100": 0.9170082867477778, "naucs_at_1_max": 0.5814659623829144, "naucs_at_1_std": -0.27243111767323397, "naucs_at_1_diff1": 0.9342450486070346, "naucs_at_3_max": 0.6913793559267872, "naucs_at_3_std": -0.13802507806435776, "naucs_at_3_diff1": 0.9039765183200388, "naucs_at_5_max": 0.7507333814600751, "naucs_at_5_std": -0.03636583769514349, "naucs_at_5_diff1": 0.8951277803578926, "naucs_at_10_max": 0.8635520027898517, "naucs_at_10_std": 0.11091336093757678, "naucs_at_10_diff1": 0.9497686715135494, "naucs_at_20_max": 0.8949615589988815, "naucs_at_20_std": 0.3466711461677555, "naucs_at_20_diff1": 0.9477594183740975, "naucs_at_50_max": 0.9183327616354471, "naucs_at_50_std": 0.3834113671390042, "naucs_at_50_diff1": 0.9738797091870569, "naucs_at_100_max": 1.0, "naucs_at_100_std": 0.35795178181150633, "naucs_at_100_diff1": 1.0 }, "./data_dir/eval_vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.836, "ndcg_at_3": 0.87881, "ndcg_at_5": 0.88716, "ndcg_at_10": 0.895, "ndcg_at_20": 0.89903, "ndcg_at_50": 0.90333, "ndcg_at_100": 0.90429, "map_at_1": 0.836, "map_at_3": 0.86867, "map_at_5": 0.87337, "map_at_10": 0.87664, "map_at_20": 0.87774, "map_at_50": 0.8784, "map_at_100": 0.87848, "recall_at_1": 0.836, "recall_at_3": 0.908, "recall_at_5": 0.928, "recall_at_10": 0.952, "recall_at_20": 0.968, "recall_at_50": 0.99, "recall_at_100": 0.996, "precision_at_1": 0.836, "precision_at_3": 0.30267, "precision_at_5": 0.1856, "precision_at_10": 0.0952, "precision_at_20": 0.0484, "precision_at_50": 0.0198, "precision_at_100": 0.00996, "mrr_at_1": 0.838, "mrr_at_3": 0.87, "mrr_at_5": 0.8745, "mrr_at_10": 0.8771547619047619, "mrr_at_20": 0.8784180427415722, "mrr_at_50": 0.8790759890841897, "mrr_at_100": 0.8791544641575035, "naucs_at_1_max": 0.7807648004607524, "naucs_at_1_std": -0.0661731179957274, "naucs_at_1_diff1": 0.9396162799068893, "naucs_at_3_max": 0.7405817399423532, "naucs_at_3_std": -0.17763772175536696, "naucs_at_3_diff1": 0.9087098607558949, "naucs_at_5_max": 0.7940268700072634, "naucs_at_5_std": -0.09364301276066404, "naucs_at_5_diff1": 0.9055918663761803, "naucs_at_10_max": 0.8829948646125115, "naucs_at_10_std": 0.11712184873949233, "naucs_at_10_diff1": 0.9087690631808285, "naucs_at_20_max": 0.9088176937441638, "naucs_at_20_std": 0.19858776844070428, "naucs_at_20_diff1": 0.9234068627450955, "naucs_at_50_max": 0.9738562091503188, "naucs_at_50_std": 0.4563025210083932, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 0.9346405228758466, "naucs_at_100_std": 0.0793650793650251, "naucs_at_100_diff1": 1.0 }, "./data_dir/eval_vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.83929, "ndcg_at_3": 0.88201, "ndcg_at_5": 0.88801, "ndcg_at_10": 0.89498, "ndcg_at_20": 0.90231, "ndcg_at_50": 0.90729, "ndcg_at_100": 0.90729, "map_at_1": 0.83929, "map_at_3": 0.87202, "map_at_5": 0.87542, "map_at_10": 0.87831, "map_at_20": 0.88038, "map_at_50": 0.88119, "map_at_100": 0.88119, "recall_at_1": 0.83929, "recall_at_3": 0.91071, "recall_at_5": 0.925, "recall_at_10": 0.94643, "recall_at_20": 0.975, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.83929, "precision_at_3": 0.30357, "precision_at_5": 0.185, "precision_at_10": 0.09464, "precision_at_20": 0.04875, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8357142857142857, "mrr_at_3": 0.8702380952380953, "mrr_at_5": 0.8736309523809525, "mrr_at_10": 0.8765221088435374, "mrr_at_20": 0.8786091537430822, "mrr_at_50": 0.8794319869910731, "mrr_at_100": 0.8794319869910731, "naucs_at_1_max": 0.4638982865873631, "naucs_at_1_std": 0.12973916839463007, "naucs_at_1_diff1": 0.8660045836516428, "naucs_at_3_max": 0.5223155929038286, "naucs_at_3_std": 0.2235854341736696, "naucs_at_3_diff1": 0.8146778711484597, "naucs_at_5_max": 0.5351029300609133, "naucs_at_5_std": 0.21993241741140748, "naucs_at_5_diff1": 0.7988306433684574, "naucs_at_10_max": 0.49181450357920947, "naucs_at_10_std": 0.2525365701836297, "naucs_at_10_diff1": 0.7532212885154057, "naucs_at_20_max": 0.8102574363078491, "naucs_at_20_std": 0.6819394424436439, "naucs_at_20_diff1": 0.715286114445777, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 } } }