| { | |
| "metadata": { | |
| "timestamp": "2025-03-12T16:10:17.229454", | |
| "vidore_benchmark_version": "5.0.1.dev5+g1548c2d.d20250312" | |
| }, | |
| "metrics": { | |
| "vidore/restaurant_esg_reports_beir":{"ndcg_at_1": 0.51923, "ndcg_at_3": 0.58018, "ndcg_at_5": 0.63101, "ndcg_at_10": 0.65274, "ndcg_at_20": 0.67047, "ndcg_at_50": 0.69998, "ndcg_at_100": 0.70624, "map_at_1": 0.39872, "map_at_3": 0.50689, "map_at_5": 0.55161, "map_at_10": 0.57265, "map_at_20": 0.57995, "map_at_50": 0.58983, "map_at_100": 0.59139, "recall_at_1": 0.39872, "recall_at_3": 0.58333, "recall_at_5": 0.71781, "recall_at_10": 0.77967, "recall_at_20": 0.83563, "recall_at_50": 0.93341, "recall_at_100": 0.9592, "precision_at_1": 0.51923, "precision_at_3": 0.3141, "precision_at_5": 0.25385, "precision_at_10": 0.14423, "precision_at_20": 0.08173, "precision_at_50": 0.04154, "precision_at_100": 0.02192, "mrr_at_1": 0.5192307692307693, "mrr_at_3": 0.625, "mrr_at_5": 0.6567307692307692, "mrr_at_10": 0.6616147741147741, "mrr_at_20": 0.6659385753135751, "mrr_at_50": 0.6668543262293262, "mrr_at_100": 0.6668543262293262, "naucs_at_1_max": 0.5694483724614683, "naucs_at_1_std": 0.3722365643641625, "naucs_at_1_diff1": 0.6630923090626469, "naucs_at_3_max": 0.4284312717311841, "naucs_at_3_std": 0.3043188529887843, "naucs_at_3_diff1": 0.23222932871139657, "naucs_at_5_max": 0.1957258984917592, "naucs_at_5_std": 0.21172075191616438, "naucs_at_5_diff1": 0.12899378896152203, "naucs_at_10_max": 0.1911079264065253, "naucs_at_10_std": 0.18818758953070164, "naucs_at_10_diff1": 0.11397146879551144, "naucs_at_20_max": 0.12239979011786752, "naucs_at_20_std": 0.23716115395048135, "naucs_at_20_diff1": -0.004465965987469208, "naucs_at_50_max": 0.024713092790819645, "naucs_at_50_std": 0.23994428769410767, "naucs_at_50_diff1": -0.11651821615679771, "naucs_at_100_max": 0.0124502351274068, "naucs_at_100_std": 0.26711349464594386, "naucs_at_100_diff1": -0.1312476583819651}, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { | |
| "ndcg_at_1": 0.5875, | |
| "ndcg_at_3": 0.57886, | |
| "ndcg_at_5": 0.60632, | |
| "ndcg_at_10": 0.64636, | |
| "ndcg_at_20": 0.67153, | |
| "ndcg_at_50": 0.69612, | |
| "ndcg_at_100": 0.70593, | |
| "map_at_1": 0.35444, | |
| "map_at_3": 0.47216, | |
| "map_at_5": 0.51385, | |
| "map_at_10": 0.5521, | |
| "map_at_20": 0.56741, | |
| "map_at_50": 0.57534, | |
| "map_at_100": 0.57786, | |
| "recall_at_1": 0.35444, | |
| "recall_at_3": 0.53706, | |
| "recall_at_5": 0.63063, | |
| "recall_at_10": 0.746, | |
| "recall_at_20": 0.81806, | |
| "recall_at_50": 0.89598, | |
| "recall_at_100": 0.92916, | |
| "precision_at_1": 0.5875, | |
| "precision_at_3": 0.35, | |
| "precision_at_5": 0.27375, | |
| "precision_at_10": 0.17938, | |
| "precision_at_20": 0.10406, | |
| "precision_at_50": 0.04975, | |
| "precision_at_100": 0.02712, | |
| "mrr_at_1": 0.5875, | |
| "mrr_at_3": 0.6812499999999998, | |
| "mrr_at_5": 0.6974999999999999, | |
| "mrr_at_10": 0.7047966269841268, | |
| "mrr_at_20": 0.7070646769166504, | |
| "mrr_at_50": 0.708239325858248, | |
| "mrr_at_100": 0.708239325858248, | |
| "naucs_at_1_max": 0.31355746190850764, | |
| "naucs_at_1_std": 0.04934974666642292, | |
| "naucs_at_1_diff1": 0.32442245431764555, | |
| "naucs_at_3_max": 0.25327223124789267, | |
| "naucs_at_3_std": 0.05050118014897422, | |
| "naucs_at_3_diff1": -0.06632845947133419, | |
| "naucs_at_5_max": 0.1851610600189157, | |
| "naucs_at_5_std": 0.058061097188027676, | |
| "naucs_at_5_diff1": -0.22026563034766006, | |
| "naucs_at_10_max": 0.15422295635424052, | |
| "naucs_at_10_std": 0.14164030177658773, | |
| "naucs_at_10_diff1": -0.27788248939794785, | |
| "naucs_at_20_max": 0.16130977100298727, | |
| "naucs_at_20_std": 0.18430855044448158, | |
| "naucs_at_20_diff1": -0.280205660946133, | |
| "naucs_at_50_max": 0.16350693053715493, | |
| "naucs_at_50_std": 0.1988648228596469, | |
| "naucs_at_50_diff1": -0.29375789506115757, | |
| "naucs_at_100_max": 0.12559087058175492, | |
| "naucs_at_100_std": 0.1908762696794704, | |
| "naucs_at_100_diff1": -0.29440996763870275 | |
| }, | |
| "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { | |
| "ndcg_at_1": 0.63793, | |
| "ndcg_at_3": 0.64059, | |
| "ndcg_at_5": 0.61184, | |
| "ndcg_at_10": 0.57183, | |
| "ndcg_at_20": 0.57051, | |
| "ndcg_at_50": 0.63089, | |
| "ndcg_at_100": 0.6723, | |
| "map_at_1": 0.07983, | |
| "map_at_3": 0.17978, | |
| "map_at_5": 0.2441, | |
| "map_at_10": 0.31313, | |
| "map_at_20": 0.35443, | |
| "map_at_50": 0.40694, | |
| "map_at_100": 0.43529, | |
| "recall_at_1": 0.07983, | |
| "recall_at_3": 0.22913, | |
| "recall_at_5": 0.33164, | |
| "recall_at_10": 0.46515, | |
| "recall_at_20": 0.5659, | |
| "recall_at_50": 0.75345, | |
| "recall_at_100": 0.87683, | |
| "precision_at_1": 0.63793, | |
| "precision_at_3": 0.61494, | |
| "precision_at_5": 0.55172, | |
| "precision_at_10": 0.41552, | |
| "precision_at_20": 0.29138, | |
| "precision_at_50": 0.18586, | |
| "precision_at_100": 0.12259, | |
| "mrr_at_1": 0.6379310344827587, | |
| "mrr_at_3": 0.7442528735632185, | |
| "mrr_at_5": 0.7563218390804599, | |
| "mrr_at_10": 0.7609195402298853, | |
| "mrr_at_20": 0.7622458001768349, | |
| "mrr_at_50": 0.7622458001768349, | |
| "mrr_at_100": 0.7622458001768349, | |
| "naucs_at_1_max": 0.419786473109018, | |
| "naucs_at_1_std": 0.07500623692548537, | |
| "naucs_at_1_diff1": 0.4755949058401351, | |
| "naucs_at_3_max": 0.4608801956792379, | |
| "naucs_at_3_std": 0.2349462202876775, | |
| "naucs_at_3_diff1": 0.31316278322181995, | |
| "naucs_at_5_max": 0.4690855783002276, | |
| "naucs_at_5_std": 0.2973952058315693, | |
| "naucs_at_5_diff1": 0.16569197052909665, | |
| "naucs_at_10_max": 0.397719696512456, | |
| "naucs_at_10_std": 0.20885173965188283, | |
| "naucs_at_10_diff1": 0.1402583115094283, | |
| "naucs_at_20_max": 0.32542799732914507, | |
| "naucs_at_20_std": 0.2747074671470978, | |
| "naucs_at_20_diff1": 0.13356908650626761, | |
| "naucs_at_50_max": 0.23125915204560868, | |
| "naucs_at_50_std": 0.1985419001386752, | |
| "naucs_at_50_diff1": 0.1130455594831512, | |
| "naucs_at_100_max": 0.1664221082762526, | |
| "naucs_at_100_std": 0.10618063335238329, | |
| "naucs_at_100_diff1": 0.1352860324631496 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0": { | |
| "ndcg_at_1": 0.47368, | |
| "ndcg_at_3": 0.47849, | |
| "ndcg_at_5": 0.50343, | |
| "ndcg_at_10": 0.5406, | |
| "ndcg_at_20": 0.5836, | |
| "ndcg_at_50": 0.61317, | |
| "ndcg_at_100": 0.62789, | |
| "map_at_1": 0.24712, | |
| "map_at_3": 0.35036, | |
| "map_at_5": 0.39048, | |
| "map_at_10": 0.42533, | |
| "map_at_20": 0.44764, | |
| "map_at_50": 0.46192, | |
| "map_at_100": 0.46709, | |
| "recall_at_1": 0.24712, | |
| "recall_at_3": 0.4477, | |
| "recall_at_5": 0.534, | |
| "recall_at_10": 0.65675, | |
| "recall_at_20": 0.79217, | |
| "recall_at_50": 0.87327, | |
| "recall_at_100": 0.92452, | |
| "precision_at_1": 0.47368, | |
| "precision_at_3": 0.30994, | |
| "precision_at_5": 0.24912, | |
| "precision_at_10": 0.16667, | |
| "precision_at_20": 0.10965, | |
| "precision_at_50": 0.0586, | |
| "precision_at_100": 0.03368, | |
| "mrr_at_1": 0.47368421052631576, | |
| "mrr_at_3": 0.5935672514619882, | |
| "mrr_at_5": 0.6058479532163742, | |
| "mrr_at_10": 0.6139376218323586, | |
| "mrr_at_20": 0.618968190678717, | |
| "mrr_at_50": 0.618968190678717, | |
| "mrr_at_100": 0.6191509392167286, | |
| "naucs_at_1_max": 0.22880110629191316, | |
| "naucs_at_1_std": 0.4807654764575437, | |
| "naucs_at_1_diff1": -0.010719521336880587, | |
| "naucs_at_3_max": 0.22255470892079213, | |
| "naucs_at_3_std": 0.26274414406533403, | |
| "naucs_at_3_diff1": 0.19901703943731583, | |
| "naucs_at_5_max": 0.08307200612000629, | |
| "naucs_at_5_std": 0.3427446267225359, | |
| "naucs_at_5_diff1": 0.03454799044188057, | |
| "naucs_at_10_max": 0.006208463905274427, | |
| "naucs_at_10_std": 0.26435566331213406, | |
| "naucs_at_10_diff1": -0.0015237809655790574, | |
| "naucs_at_20_max": 0.04503846394507633, | |
| "naucs_at_20_std": 0.2139553060662944, | |
| "naucs_at_20_diff1": 0.018201732846277442, | |
| "naucs_at_50_max": 0.11296962958977845, | |
| "naucs_at_50_std": 0.2632828869514633, | |
| "naucs_at_50_diff1": -0.08837586390454706, | |
| "naucs_at_100_max": 0.11868484014177574, | |
| "naucs_at_100_std": 0.2734455942211179, | |
| "naucs_at_100_diff1": -0.13438673026134765 | |
| }, | |
| "vidore/synthetic_axa_filtered_v1.0": { | |
| "ndcg_at_1": 0.72222, | |
| "ndcg_at_3": 0.72275, | |
| "ndcg_at_5": 0.68826, | |
| "ndcg_at_10": 0.69213, | |
| "ndcg_at_20": 0.71771, | |
| "ndcg_at_50": 0.77886, | |
| "ndcg_at_100": 0.79636, | |
| "map_at_1": 0.40655, | |
| "map_at_3": 0.4985, | |
| "map_at_5": 0.53149, | |
| "map_at_10": 0.58441, | |
| "map_at_20": 0.61212, | |
| "map_at_50": 0.64312, | |
| "map_at_100": 0.64864, | |
| "recall_at_1": 0.40655, | |
| "recall_at_3": 0.55187, | |
| "recall_at_5": 0.6011, | |
| "recall_at_10": 0.69041, | |
| "recall_at_20": 0.76244, | |
| "recall_at_50": 0.92875, | |
| "recall_at_100": 0.99383, | |
| "precision_at_1": 0.72222, | |
| "precision_at_3": 0.46296, | |
| "precision_at_5": 0.34444, | |
| "precision_at_10": 0.23889, | |
| "precision_at_20": 0.15556, | |
| "precision_at_50": 0.08778, | |
| "precision_at_100": 0.04722, | |
| "mrr_at_1": 0.7222222222222222, | |
| "mrr_at_3": 0.8333333333333334, | |
| "mrr_at_5": 0.8333333333333334, | |
| "mrr_at_10": 0.8333333333333334, | |
| "mrr_at_20": 0.8333333333333334, | |
| "mrr_at_50": 0.834625322997416, | |
| "mrr_at_100": 0.834625322997416, | |
| "naucs_at_1_max": 0.284337842431283, | |
| "naucs_at_1_std": -0.5569014279726879, | |
| "naucs_at_1_diff1": 0.9294297019639159, | |
| "naucs_at_3_max": 0.4036627064656008, | |
| "naucs_at_3_std": -0.258134268334631, | |
| "naucs_at_3_diff1": 0.03319977085307361, | |
| "naucs_at_5_max": 0.3391788551975161, | |
| "naucs_at_5_std": -0.13525790110164213, | |
| "naucs_at_5_diff1": -0.08545811027367776, | |
| "naucs_at_10_max": 0.17205834819466667, | |
| "naucs_at_10_std": -0.16161238583793947, | |
| "naucs_at_10_diff1": -0.12294732613259658, | |
| "naucs_at_20_max": 0.019221817231354917, | |
| "naucs_at_20_std": -0.18190133460121213, | |
| "naucs_at_20_diff1": -0.26196318223082454, | |
| "naucs_at_50_max": -0.1775802959975011, | |
| "naucs_at_50_std": -0.29278726428875046, | |
| "naucs_at_50_diff1": -0.3578106924688144, | |
| "naucs_at_100_max": -0.11947266465729971, | |
| "naucs_at_100_std": -0.30339673690269625, | |
| "naucs_at_100_diff1": -0.3387677917417156 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.47368, | |
| "ndcg_at_3": 0.48112, | |
| "ndcg_at_5": 0.51162, | |
| "ndcg_at_10": 0.55587, | |
| "ndcg_at_20": 0.593, | |
| "ndcg_at_50": 0.62154, | |
| "ndcg_at_100": 0.63537, | |
| "map_at_1": 0.23657, | |
| "map_at_3": 0.3516, | |
| "map_at_5": 0.39581, | |
| "map_at_10": 0.43424, | |
| "map_at_20": 0.45458, | |
| "map_at_50": 0.46799, | |
| "map_at_100": 0.47342, | |
| "recall_at_1": 0.23657, | |
| "recall_at_3": 0.44464, | |
| "recall_at_5": 0.5509, | |
| "recall_at_10": 0.69046, | |
| "recall_at_20": 0.80641, | |
| "recall_at_50": 0.88589, | |
| "recall_at_100": 0.92726, | |
| "precision_at_1": 0.47368, | |
| "precision_at_3": 0.31871, | |
| "precision_at_5": 0.25526, | |
| "precision_at_10": 0.17412, | |
| "precision_at_20": 0.11096, | |
| "precision_at_50": 0.05851, | |
| "precision_at_100": 0.0339, | |
| "mrr_at_1": 0.47368421052631576, | |
| "mrr_at_3": 0.5921052631578947, | |
| "mrr_at_5": 0.6070175438596491, | |
| "mrr_at_10": 0.618662976886661, | |
| "mrr_at_20": 0.623367953282814, | |
| "mrr_at_50": 0.6237087980633872, | |
| "mrr_at_100": 0.6237544851978901, | |
| "naucs_at_1_max": 0.030121925389534895, | |
| "naucs_at_1_std": 0.22985280707732192, | |
| "naucs_at_1_diff1": 0.09815343782760363, | |
| "naucs_at_3_max": 0.08222682574140788, | |
| "naucs_at_3_std": 0.20242031488104395, | |
| "naucs_at_3_diff1": 0.043651752642660774, | |
| "naucs_at_5_max": 0.029502810692712066, | |
| "naucs_at_5_std": 0.2552530112160377, | |
| "naucs_at_5_diff1": -0.01488436104125662, | |
| "naucs_at_10_max": 0.043798500718208115, | |
| "naucs_at_10_std": 0.25656427779016705, | |
| "naucs_at_10_diff1": -0.03213615749189139, | |
| "naucs_at_20_max": 0.08483260560870946, | |
| "naucs_at_20_std": 0.22759171353041793, | |
| "naucs_at_20_diff1": -0.013226797212940839, | |
| "naucs_at_50_max": 0.145727206325556, | |
| "naucs_at_50_std": 0.2840662881602473, | |
| "naucs_at_50_diff1": -0.12024084882915537, | |
| "naucs_at_100_max": 0.16860092521171394, | |
| "naucs_at_100_std": 0.324767526336977, | |
| "naucs_at_100_diff1": -0.15998217261471917 | |
| }, | |
| "vidore/synthetic_axa_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.63889, | |
| "ndcg_at_3": 0.62932, | |
| "ndcg_at_5": 0.61003, | |
| "ndcg_at_10": 0.62475, | |
| "ndcg_at_20": 0.64803, | |
| "ndcg_at_50": 0.71389, | |
| "ndcg_at_100": 0.73221, | |
| "map_at_1": 0.33774, | |
| "map_at_3": 0.42553, | |
| "map_at_5": 0.46128, | |
| "map_at_10": 0.51109, | |
| "map_at_20": 0.5344, | |
| "map_at_50": 0.56198, | |
| "map_at_100": 0.56769, | |
| "recall_at_1": 0.33774, | |
| "recall_at_3": 0.48038, | |
| "recall_at_5": 0.55351, | |
| "recall_at_10": 0.66626, | |
| "recall_at_20": 0.72903, | |
| "recall_at_50": 0.91514, | |
| "recall_at_100": 0.97653, | |
| "precision_at_1": 0.63889, | |
| "precision_at_3": 0.40741, | |
| "precision_at_5": 0.31111, | |
| "precision_at_10": 0.21806, | |
| "precision_at_20": 0.13889, | |
| "precision_at_50": 0.08222, | |
| "precision_at_100": 0.04542, | |
| "mrr_at_1": 0.6388888888888888, | |
| "mrr_at_3": 0.75, | |
| "mrr_at_5": 0.7569444444444444, | |
| "mrr_at_10": 0.7609126984126985, | |
| "mrr_at_20": 0.7621753246753247, | |
| "mrr_at_50": 0.7634990122224704, | |
| "mrr_at_100": 0.7634990122224704, | |
| "naucs_at_1_max": 0.4716487479189039, | |
| "naucs_at_1_std": -0.006727437637346147, | |
| "naucs_at_1_diff1": 0.6581008709633844, | |
| "naucs_at_3_max": 0.4695980071561003, | |
| "naucs_at_3_std": 0.07699949409760401, | |
| "naucs_at_3_diff1": 0.37046512198112674, | |
| "naucs_at_5_max": 0.39746806745579205, | |
| "naucs_at_5_std": 0.08389923773194792, | |
| "naucs_at_5_diff1": 0.2667449430996483, | |
| "naucs_at_10_max": 0.27925046935465064, | |
| "naucs_at_10_std": 0.04493147311300899, | |
| "naucs_at_10_diff1": 0.14879018650447173, | |
| "naucs_at_20_max": 0.1724123702015283, | |
| "naucs_at_20_std": -0.0406874705309399, | |
| "naucs_at_20_diff1": 0.005456808130550691, | |
| "naucs_at_50_max": 0.053840240229350286, | |
| "naucs_at_50_std": -0.15038250749984536, | |
| "naucs_at_50_diff1": -0.15099178882024658, | |
| "naucs_at_100_max": 0.017995420870417105, | |
| "naucs_at_100_std": -0.1816075038440127, | |
| "naucs_at_100_diff1": -0.18370322898832045 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { | |
| "ndcg_at_1": 0.53438, | |
| "ndcg_at_3": 0.54593, | |
| "ndcg_at_5": 0.56864, | |
| "ndcg_at_10": 0.60603, | |
| "ndcg_at_20": 0.63584, | |
| "ndcg_at_50": 0.66155, | |
| "ndcg_at_100": 0.67448, | |
| "map_at_1": 0.3256, | |
| "map_at_3": 0.43974, | |
| "map_at_5": 0.47749, | |
| "map_at_10": 0.51301, | |
| "map_at_20": 0.52876, | |
| "map_at_50": 0.53796, | |
| "map_at_100": 0.54086, | |
| "recall_at_1": 0.3256, | |
| "recall_at_3": 0.50788, | |
| "recall_at_5": 0.59866, | |
| "recall_at_10": 0.70913, | |
| "recall_at_20": 0.79605, | |
| "recall_at_50": 0.87472, | |
| "recall_at_100": 0.92428, | |
| "precision_at_1": 0.53438, | |
| "precision_at_3": 0.3349, | |
| "precision_at_5": 0.2575, | |
| "precision_at_10": 0.16875, | |
| "precision_at_20": 0.1007, | |
| "precision_at_50": 0.04887, | |
| "precision_at_100": 0.027, | |
| "mrr_at_1": 0.534375, | |
| "mrr_at_3": 0.6351562499999994, | |
| "mrr_at_5": 0.6516406249999992, | |
| "mrr_at_10": 0.6609145585317455, | |
| "mrr_at_20": 0.664847090359921, | |
| "mrr_at_50": 0.6655434024683474, | |
| "mrr_at_100": 0.6656955962320235, | |
| "naucs_at_1_max": 0.289841750710978, | |
| "naucs_at_1_std": -0.039288288171653626, | |
| "naucs_at_1_diff1": 0.39128614768495107, | |
| "naucs_at_3_max": 0.18877375406489166, | |
| "naucs_at_3_std": 0.028383723285413855, | |
| "naucs_at_3_diff1": -0.000027337444719206833, | |
| "naucs_at_5_max": 0.15432591818453287, | |
| "naucs_at_5_std": 0.02995110480740528, | |
| "naucs_at_5_diff1": -0.11005803971334709, | |
| "naucs_at_10_max": 0.13088890322554914, | |
| "naucs_at_10_std": 0.06919596389023963, | |
| "naucs_at_10_diff1": -0.17785912723881914, | |
| "naucs_at_20_max": 0.1468487573626851, | |
| "naucs_at_20_std": 0.14193307673103725, | |
| "naucs_at_20_diff1": -0.20865497781935535, | |
| "naucs_at_50_max": 0.13335503376393565, | |
| "naucs_at_50_std": 0.13252192227124457, | |
| "naucs_at_50_diff1": -0.22225698776995703, | |
| "naucs_at_100_max": 0.0920178712104123, | |
| "naucs_at_100_std": 0.11787379937051319, | |
| "naucs_at_100_diff1": -0.22789195906486828 | |
| }, | |
| "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.57759, | |
| "ndcg_at_3": 0.56007, | |
| "ndcg_at_5": 0.52841, | |
| "ndcg_at_10": 0.50961, | |
| "ndcg_at_20": 0.52797, | |
| "ndcg_at_50": 0.59657, | |
| "ndcg_at_100": 0.63916, | |
| "map_at_1": 0.07684, | |
| "map_at_3": 0.16481, | |
| "map_at_5": 0.20962, | |
| "map_at_10": 0.26965, | |
| "map_at_20": 0.31679, | |
| "map_at_50": 0.3705, | |
| "map_at_100": 0.39716, | |
| "recall_at_1": 0.07684, | |
| "recall_at_3": 0.20767, | |
| "recall_at_5": 0.2827, | |
| "recall_at_10": 0.40917, | |
| "recall_at_20": 0.5326, | |
| "recall_at_50": 0.73621, | |
| "recall_at_100": 0.86845, | |
| "precision_at_1": 0.57759, | |
| "precision_at_3": 0.52586, | |
| "precision_at_5": 0.46552, | |
| "precision_at_10": 0.37241, | |
| "precision_at_20": 0.27974, | |
| "precision_at_50": 0.18276, | |
| "precision_at_100": 0.12022, | |
| "mrr_at_1": 0.5775862068965517, | |
| "mrr_at_3": 0.682471264367816, | |
| "mrr_at_5": 0.7003591954022986, | |
| "mrr_at_10": 0.7084325396825395, | |
| "mrr_at_20": 0.7095603211443726, | |
| "mrr_at_50": 0.7103997141136704, | |
| "mrr_at_100": 0.7104842306789173, | |
| "naucs_at_1_max": 0.386719550550633, | |
| "naucs_at_1_std": 0.03964188186567574, | |
| "naucs_at_1_diff1": 0.29324581025971, | |
| "naucs_at_3_max": 0.3940438135481905, | |
| "naucs_at_3_std": 0.11916750908192153, | |
| "naucs_at_3_diff1": 0.11789001603532652, | |
| "naucs_at_5_max": 0.4355532765854707, | |
| "naucs_at_5_std": 0.2551718488349167, | |
| "naucs_at_5_diff1": 0.013367355141125142, | |
| "naucs_at_10_max": 0.3421752314533559, | |
| "naucs_at_10_std": 0.23601153039445807, | |
| "naucs_at_10_diff1": -0.007635368428971648, | |
| "naucs_at_20_max": 0.2981468248373431, | |
| "naucs_at_20_std": 0.25464689427959064, | |
| "naucs_at_20_diff1": -0.0018484213984256066, | |
| "naucs_at_50_max": 0.20823391530171706, | |
| "naucs_at_50_std": 0.18448677686850273, | |
| "naucs_at_50_diff1": -0.03204255205680404, | |
| "naucs_at_100_max": 0.16804217929784568, | |
| "naucs_at_100_std": 0.13259088431947955, | |
| "naucs_at_100_diff1": -0.017832844620359677 | |
| }, | |
| "vidore/arxivqa_test_subsampled": { | |
| "ndcg_at_1": 0.776, | |
| "ndcg_at_3": 0.83117, | |
| "ndcg_at_5": 0.84184, | |
| "ndcg_at_10": 0.8527, | |
| "ndcg_at_20": 0.8598, | |
| "ndcg_at_50": 0.86593, | |
| "ndcg_at_100": 0.86754, | |
| "map_at_1": 0.776, | |
| "map_at_3": 0.81833, | |
| "map_at_5": 0.82423, | |
| "map_at_10": 0.82865, | |
| "map_at_20": 0.83061, | |
| "map_at_50": 0.83167, | |
| "map_at_100": 0.83181, | |
| "recall_at_1": 0.776, | |
| "recall_at_3": 0.868, | |
| "recall_at_5": 0.894, | |
| "recall_at_10": 0.928, | |
| "recall_at_20": 0.956, | |
| "recall_at_50": 0.986, | |
| "recall_at_100": 0.996, | |
| "precision_at_1": 0.776, | |
| "precision_at_3": 0.28933, | |
| "precision_at_5": 0.1788, | |
| "precision_at_10": 0.0928, | |
| "precision_at_20": 0.0478, | |
| "precision_at_50": 0.01972, | |
| "precision_at_100": 0.00996, | |
| "mrr_at_1": 0.776, | |
| "mrr_at_3": 0.8183333333333332, | |
| "mrr_at_5": 0.8242333333333332, | |
| "mrr_at_10": 0.8286515873015872, | |
| "mrr_at_20": 0.8306089112641741, | |
| "mrr_at_50": 0.8316740970253331, | |
| "mrr_at_100": 0.8318131621644544, | |
| "naucs_at_1_max": 0.33058912801009327, | |
| "naucs_at_1_std": -0.3471366403978715, | |
| "naucs_at_1_diff1": 0.9033890402442812, | |
| "naucs_at_3_max": 0.19704584636063602, | |
| "naucs_at_3_std": -0.3672895421207954, | |
| "naucs_at_3_diff1": 0.8800149022587156, | |
| "naucs_at_5_max": 0.21286583958965521, | |
| "naucs_at_5_std": -0.3741724792787036, | |
| "naucs_at_5_diff1": 0.8638251424311815, | |
| "naucs_at_10_max": 0.12077238302728274, | |
| "naucs_at_10_std": -0.3839739599543562, | |
| "naucs_at_10_diff1": 0.8619540408756106, | |
| "naucs_at_20_max": 0.15711739241150968, | |
| "naucs_at_20_std": -0.269353195823785, | |
| "naucs_at_20_diff1": 0.9188311688311718, | |
| "naucs_at_50_max": 0.012538348672796787, | |
| "naucs_at_50_std": -0.17046818727492316, | |
| "naucs_at_50_diff1": 0.9416433239962615, | |
| "naucs_at_100_max": -0.3674136321195193, | |
| "naucs_at_100_std": 0.21825396825390833, | |
| "naucs_at_100_diff1": 1.0 | |
| }, | |
| "vidore/docvqa_test_subsampled": { | |
| "ndcg_at_1": 0.47672, | |
| "ndcg_at_3": 0.55403, | |
| "ndcg_at_5": 0.57789, | |
| "ndcg_at_10": 0.59341, | |
| "ndcg_at_20": 0.60923, | |
| "ndcg_at_50": 0.62426, | |
| "ndcg_at_100": 0.63194, | |
| "map_at_1": 0.47672, | |
| "map_at_3": 0.53548, | |
| "map_at_5": 0.54878, | |
| "map_at_10": 0.55547, | |
| "map_at_20": 0.55988, | |
| "map_at_50": 0.56214, | |
| "map_at_100": 0.56285, | |
| "recall_at_1": 0.47672, | |
| "recall_at_3": 0.60754, | |
| "recall_at_5": 0.66519, | |
| "recall_at_10": 0.71175, | |
| "recall_at_20": 0.77384, | |
| "recall_at_50": 0.85144, | |
| "recall_at_100": 0.898, | |
| "precision_at_1": 0.47672, | |
| "precision_at_3": 0.20251, | |
| "precision_at_5": 0.13304, | |
| "precision_at_10": 0.07118, | |
| "precision_at_20": 0.03869, | |
| "precision_at_50": 0.01703, | |
| "precision_at_100": 0.00898, | |
| "mrr_at_1": 0.47671840354767187, | |
| "mrr_at_3": 0.5354767184035478, | |
| "mrr_at_5": 0.5487804878048781, | |
| "mrr_at_10": 0.5554746066941187, | |
| "mrr_at_20": 0.5598837982651386, | |
| "mrr_at_50": 0.5621395811039165, | |
| "mrr_at_100": 0.562853769260598, | |
| "naucs_at_1_max": -0.34988732060075106, | |
| "naucs_at_1_std": -0.0024823362084227295, | |
| "naucs_at_1_diff1": 0.7709257756426611, | |
| "naucs_at_3_max": -0.3848072440519266, | |
| "naucs_at_3_std": 0.04900442240358232, | |
| "naucs_at_3_diff1": 0.6684105207204192, | |
| "naucs_at_5_max": -0.4729915658738238, | |
| "naucs_at_5_std": 0.051502600367980475, | |
| "naucs_at_5_diff1": 0.635314656852817, | |
| "naucs_at_10_max": -0.4755655825410273, | |
| "naucs_at_10_std": 0.14092635748512966, | |
| "naucs_at_10_diff1": 0.6142413640583799, | |
| "naucs_at_20_max": -0.5013121261464005, | |
| "naucs_at_20_std": 0.26884721841803433, | |
| "naucs_at_20_diff1": 0.5864223996381905, | |
| "naucs_at_50_max": -0.46761445911554894, | |
| "naucs_at_50_std": 0.5204115851881066, | |
| "naucs_at_50_diff1": 0.5754658330685111, | |
| "naucs_at_100_max": -0.45463665233128225, | |
| "naucs_at_100_std": 0.6231821590075602, | |
| "naucs_at_100_diff1": 0.5438810608835282 | |
| }, | |
| "vidore/infovqa_test_subsampled": { | |
| "ndcg_at_1": 0.80162, | |
| "ndcg_at_3": 0.86102, | |
| "ndcg_at_5": 0.87252, | |
| "ndcg_at_10": 0.88185, | |
| "ndcg_at_20": 0.88587, | |
| "ndcg_at_50": 0.88753, | |
| "ndcg_at_100": 0.88887, | |
| "map_at_1": 0.80162, | |
| "map_at_3": 0.84717, | |
| "map_at_5": 0.85344, | |
| "map_at_10": 0.85739, | |
| "map_at_20": 0.85845, | |
| "map_at_50": 0.85874, | |
| "map_at_100": 0.85886, | |
| "recall_at_1": 0.80162, | |
| "recall_at_3": 0.90081, | |
| "recall_at_5": 0.92915, | |
| "recall_at_10": 0.95749, | |
| "recall_at_20": 0.97368, | |
| "recall_at_50": 0.98178, | |
| "recall_at_100": 0.98988, | |
| "precision_at_1": 0.80162, | |
| "precision_at_3": 0.30027, | |
| "precision_at_5": 0.18583, | |
| "precision_at_10": 0.09575, | |
| "precision_at_20": 0.04868, | |
| "precision_at_50": 0.01964, | |
| "precision_at_100": 0.0099, | |
| "mrr_at_1": 0.8016194331983806, | |
| "mrr_at_3": 0.8471659919028339, | |
| "mrr_at_5": 0.8534412955465585, | |
| "mrr_at_10": 0.8573934837092728, | |
| "mrr_at_20": 0.8584495144744448, | |
| "mrr_at_50": 0.8587416976822032, | |
| "mrr_at_100": 0.8588638098799428, | |
| "naucs_at_1_max": 0.36644356235169917, | |
| "naucs_at_1_std": -0.0621555111999374, | |
| "naucs_at_1_diff1": 0.9156953149702299, | |
| "naucs_at_3_max": 0.31921758147803636, | |
| "naucs_at_3_std": -0.07179256055822544, | |
| "naucs_at_3_diff1": 0.8529538901636119, | |
| "naucs_at_5_max": 0.4367975086754174, | |
| "naucs_at_5_std": 0.2066108147068303, | |
| "naucs_at_5_diff1": 0.8413977765662696, | |
| "naucs_at_10_max": 0.4577426306876829, | |
| "naucs_at_10_std": 0.5149462893773337, | |
| "naucs_at_10_diff1": 0.8478907857538679, | |
| "naucs_at_20_max": 0.3736598448014174, | |
| "naucs_at_20_std": 0.716083799779582, | |
| "naucs_at_20_diff1": 0.8300268432831852, | |
| "naucs_at_50_max": 0.38100826473331617, | |
| "naucs_at_50_std": 0.9256067666719673, | |
| "naucs_at_50_diff1": 0.8288764514037448, | |
| "naucs_at_100_max": 0.380281489664749, | |
| "naucs_at_100_std": 0.9738797091870569, | |
| "naucs_at_100_diff1": 0.9183327616354471 | |
| }, | |
| "vidore/tabfquad_test_subsampled": { | |
| "ndcg_at_1": 0.90714, | |
| "ndcg_at_3": 0.94668, | |
| "ndcg_at_5": 0.94806, | |
| "ndcg_at_10": 0.95141, | |
| "ndcg_at_20": 0.95329, | |
| "ndcg_at_50": 0.95329, | |
| "ndcg_at_100": 0.95388, | |
| "map_at_1": 0.90714, | |
| "map_at_3": 0.9369, | |
| "map_at_5": 0.93762, | |
| "map_at_10": 0.93893, | |
| "map_at_20": 0.93948, | |
| "map_at_50": 0.93948, | |
| "map_at_100": 0.93954, | |
| "recall_at_1": 0.90714, | |
| "recall_at_3": 0.975, | |
| "recall_at_5": 0.97857, | |
| "recall_at_10": 0.98929, | |
| "recall_at_20": 0.99643, | |
| "recall_at_50": 0.99643, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.90714, | |
| "precision_at_3": 0.325, | |
| "precision_at_5": 0.19571, | |
| "precision_at_10": 0.09893, | |
| "precision_at_20": 0.04982, | |
| "precision_at_50": 0.01993, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.9071428571428571, | |
| "mrr_at_3": 0.9369047619047618, | |
| "mrr_at_5": 0.9376190476190477, | |
| "mrr_at_10": 0.9389328231292515, | |
| "mrr_at_20": 0.9394822736787022, | |
| "mrr_at_50": 0.9394822736787022, | |
| "mrr_at_100": 0.9395380772501308, | |
| "naucs_at_1_max": 0.6988436400201107, | |
| "naucs_at_1_std": 0.1637218990160183, | |
| "naucs_at_1_diff1": 0.8848128995187822, | |
| "naucs_at_3_max": 0.98132586367881, | |
| "naucs_at_3_std": 0.2895158063225239, | |
| "naucs_at_3_diff1": 0.9229691876750775, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 0.5297230003112448, | |
| "naucs_at_5_diff1": 0.910130718954251, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 0.820261437908502, | |
| "naucs_at_10_diff1": 0.9564270152505505, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 0.8692810457516478, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": 1.0, | |
| "naucs_at_50_std": 0.8692810457515607, | |
| "naucs_at_50_diff1": 1.0, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 1.0, | |
| "naucs_at_100_diff1": 1.0 | |
| }, | |
| "vidore/tatdqa_test": { | |
| "ndcg_at_1": 0.53463, | |
| "ndcg_at_3": 0.63835, | |
| "ndcg_at_5": 0.668, | |
| "ndcg_at_10": 0.70153, | |
| "ndcg_at_20": 0.71324, | |
| "ndcg_at_50": 0.72125, | |
| "ndcg_at_100": 0.72388, | |
| "map_at_1": 0.53463, | |
| "map_at_3": 0.6127, | |
| "map_at_5": 0.62922, | |
| "map_at_10": 0.64314, | |
| "map_at_20": 0.64645, | |
| "map_at_50": 0.64781, | |
| "map_at_100": 0.64806, | |
| "recall_at_1": 0.53463, | |
| "recall_at_3": 0.71264, | |
| "recall_at_5": 0.78433, | |
| "recall_at_10": 0.88761, | |
| "recall_at_20": 0.93317, | |
| "recall_at_50": 0.97266, | |
| "recall_at_100": 0.98846, | |
| "precision_at_1": 0.53463, | |
| "precision_at_3": 0.23755, | |
| "precision_at_5": 0.15687, | |
| "precision_at_10": 0.08876, | |
| "precision_at_20": 0.04666, | |
| "precision_at_50": 0.01945, | |
| "precision_at_100": 0.00988, | |
| "mrr_at_1": 0.5328068043742406, | |
| "mrr_at_3": 0.6120899149453227, | |
| "mrr_at_5": 0.628736330498179, | |
| "mrr_at_10": 0.6424352446527422, | |
| "mrr_at_20": 0.6457490447027197, | |
| "mrr_at_50": 0.6471051191468972, | |
| "mrr_at_100": 0.6473555641491229, | |
| "naucs_at_1_max": 0.004770766180126278, | |
| "naucs_at_1_std": -0.14754760705264022, | |
| "naucs_at_1_diff1": 0.6935989861691055, | |
| "naucs_at_3_max": -0.0016008692400043853, | |
| "naucs_at_3_std": -0.13025322915899146, | |
| "naucs_at_3_diff1": 0.6105721528628484, | |
| "naucs_at_5_max": 0.010908860308784006, | |
| "naucs_at_5_std": -0.07481967080015482, | |
| "naucs_at_5_diff1": 0.5707165994173674, | |
| "naucs_at_10_max": 0.13140741044161422, | |
| "naucs_at_10_std": 0.05754247942547455, | |
| "naucs_at_10_diff1": 0.5353376028377649, | |
| "naucs_at_20_max": 0.22540165846494245, | |
| "naucs_at_20_std": 0.2108123379388908, | |
| "naucs_at_20_diff1": 0.51950176941338, | |
| "naucs_at_50_max": 0.2798785039808347, | |
| "naucs_at_50_std": 0.6091081627491584, | |
| "naucs_at_50_diff1": 0.5097940423840385, | |
| "naucs_at_100_max": 0.41272456649348505, | |
| "naucs_at_100_std": 0.8595897297980766, | |
| "naucs_at_100_diff1": 0.580463276628437 | |
| }, | |
| "vidore/shiftproject_test": { | |
| "ndcg_at_1": 0.74, | |
| "ndcg_at_3": 0.8394, | |
| "ndcg_at_5": 0.85575, | |
| "ndcg_at_10": 0.86589, | |
| "ndcg_at_20": 0.86868, | |
| "ndcg_at_50": 0.87054, | |
| "ndcg_at_100": 0.87054, | |
| "map_at_1": 0.74, | |
| "map_at_3": 0.815, | |
| "map_at_5": 0.824, | |
| "map_at_10": 0.82844, | |
| "map_at_20": 0.82935, | |
| "map_at_50": 0.8296, | |
| "map_at_100": 0.8296, | |
| "recall_at_1": 0.74, | |
| "recall_at_3": 0.91, | |
| "recall_at_5": 0.95, | |
| "recall_at_10": 0.98, | |
| "recall_at_20": 0.99, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.74, | |
| "precision_at_3": 0.30333, | |
| "precision_at_5": 0.19, | |
| "precision_at_10": 0.098, | |
| "precision_at_20": 0.0495, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.74, | |
| "mrr_at_3": 0.8149999999999998, | |
| "mrr_at_5": 0.8240000000000001, | |
| "mrr_at_10": 0.8284444444444445, | |
| "mrr_at_20": 0.8293535353535354, | |
| "mrr_at_50": 0.8296035353535354, | |
| "mrr_at_100": 0.8296035353535354, | |
| "naucs_at_1_max": 0.22610599732370926, | |
| "naucs_at_1_std": -0.32918373139775386, | |
| "naucs_at_1_diff1": 0.5869186164389112, | |
| "naucs_at_3_max": 0.5146280734516033, | |
| "naucs_at_3_std": -0.4240585122938067, | |
| "naucs_at_3_diff1": 0.5019192862330117, | |
| "naucs_at_5_max": 0.43902894491130423, | |
| "naucs_at_5_std": -0.038001867413624234, | |
| "naucs_at_5_diff1": 0.3900093370681662, | |
| "naucs_at_10_max": 0.24042950513538955, | |
| "naucs_at_10_std": 0.35807656395892185, | |
| "naucs_at_10_diff1": 0.27544351073763346, | |
| "naucs_at_20_max": 0.35807656395891135, | |
| "naucs_at_20_std": 0.35807656395891135, | |
| "naucs_at_20_diff1": -0.1713352007469681, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_artificial_intelligence_test": { | |
| "ndcg_at_1": 0.93, | |
| "ndcg_at_3": 0.97155, | |
| "ndcg_at_5": 0.97155, | |
| "ndcg_at_10": 0.97155, | |
| "ndcg_at_20": 0.97155, | |
| "ndcg_at_50": 0.97155, | |
| "ndcg_at_100": 0.97155, | |
| "map_at_1": 0.93, | |
| "map_at_3": 0.96167, | |
| "map_at_5": 0.96167, | |
| "map_at_10": 0.96167, | |
| "map_at_20": 0.96167, | |
| "map_at_50": 0.96167, | |
| "map_at_100": 0.96167, | |
| "recall_at_1": 0.93, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.93, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.93, | |
| "mrr_at_3": 0.9616666666666667, | |
| "mrr_at_5": 0.9616666666666667, | |
| "mrr_at_10": 0.9616666666666667, | |
| "mrr_at_20": 0.9616666666666667, | |
| "mrr_at_50": 0.9616666666666667, | |
| "mrr_at_100": 0.9616666666666667, | |
| "naucs_at_1_max": 0.23656129118313848, | |
| "naucs_at_1_std": -0.8155929038282005, | |
| "naucs_at_1_diff1": 0.9813258636788056, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_energy_test": { | |
| "ndcg_at_1": 0.86, | |
| "ndcg_at_3": 0.90286, | |
| "ndcg_at_5": 0.91921, | |
| "ndcg_at_10": 0.92592, | |
| "ndcg_at_20": 0.92592, | |
| "ndcg_at_50": 0.92794, | |
| "ndcg_at_100": 0.92794, | |
| "map_at_1": 0.86, | |
| "map_at_3": 0.89333, | |
| "map_at_5": 0.90233, | |
| "map_at_10": 0.90525, | |
| "map_at_20": 0.90525, | |
| "map_at_50": 0.90558, | |
| "map_at_100": 0.90558, | |
| "recall_at_1": 0.86, | |
| "recall_at_3": 0.93, | |
| "recall_at_5": 0.97, | |
| "recall_at_10": 0.99, | |
| "recall_at_20": 0.99, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.86, | |
| "precision_at_3": 0.31, | |
| "precision_at_5": 0.194, | |
| "precision_at_10": 0.099, | |
| "precision_at_20": 0.0495, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.86, | |
| "mrr_at_3": 0.8933333333333334, | |
| "mrr_at_5": 0.9023333333333333, | |
| "mrr_at_10": 0.9052499999999999, | |
| "mrr_at_20": 0.9052499999999999, | |
| "mrr_at_50": 0.9055833333333332, | |
| "mrr_at_100": 0.9055833333333332, | |
| "naucs_at_1_max": 0.5179437439379241, | |
| "naucs_at_1_std": -0.3843702369405563, | |
| "naucs_at_1_diff1": 0.9381668283220175, | |
| "naucs_at_3_max": 0.4529811924769899, | |
| "naucs_at_3_std": -0.7859810590903017, | |
| "naucs_at_3_diff1": 0.9416433239962636, | |
| "naucs_at_5_max": 0.317149081854964, | |
| "naucs_at_5_std": -0.2983193277310949, | |
| "naucs_at_5_diff1": 0.9074074074074053, | |
| "naucs_at_10_max": -0.1713352007469681, | |
| "naucs_at_10_std": -1.7399626517273863, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": -0.1713352007469681, | |
| "naucs_at_20_std": -1.7399626517273863, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_government_reports_test": { | |
| "ndcg_at_1": 0.9, | |
| "ndcg_at_3": 0.95417, | |
| "ndcg_at_5": 0.95847, | |
| "ndcg_at_10": 0.95847, | |
| "ndcg_at_20": 0.95847, | |
| "ndcg_at_50": 0.95847, | |
| "ndcg_at_100": 0.95847, | |
| "map_at_1": 0.9, | |
| "map_at_3": 0.94167, | |
| "map_at_5": 0.94417, | |
| "map_at_10": 0.94417, | |
| "map_at_20": 0.94417, | |
| "map_at_50": 0.94417, | |
| "map_at_100": 0.94417, | |
| "recall_at_1": 0.9, | |
| "recall_at_3": 0.99, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.9, | |
| "precision_at_3": 0.33, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.9, | |
| "mrr_at_3": 0.9416666666666668, | |
| "mrr_at_5": 0.9441666666666667, | |
| "mrr_at_10": 0.9441666666666667, | |
| "mrr_at_20": 0.9441666666666667, | |
| "mrr_at_50": 0.9441666666666667, | |
| "mrr_at_100": 0.9441666666666667, | |
| "naucs_at_1_max": 0.5605508870214744, | |
| "naucs_at_1_std": -0.07072829131652612, | |
| "naucs_at_1_diff1": 0.8389355742296911, | |
| "naucs_at_3_max": 0.12278244631183229, | |
| "naucs_at_3_std": -1.7399626517274398, | |
| "naucs_at_3_diff1": 0.35807656395889226, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "vidore/syntheticDocQA_healthcare_industry_test": { | |
| "ndcg_at_1": 0.92, | |
| "ndcg_at_3": 0.96917, | |
| "ndcg_at_5": 0.96917, | |
| "ndcg_at_10": 0.96917, | |
| "ndcg_at_20": 0.96917, | |
| "ndcg_at_50": 0.96917, | |
| "ndcg_at_100": 0.96917, | |
| "map_at_1": 0.92, | |
| "map_at_3": 0.95833, | |
| "map_at_5": 0.95833, | |
| "map_at_10": 0.95833, | |
| "map_at_20": 0.95833, | |
| "map_at_50": 0.95833, | |
| "map_at_100": 0.95833, | |
| "recall_at_1": 0.92, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.92, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.92, | |
| "mrr_at_3": 0.9583333333333333, | |
| "mrr_at_5": 0.9583333333333333, | |
| "mrr_at_10": 0.9583333333333333, | |
| "mrr_at_20": 0.9583333333333333, | |
| "mrr_at_50": 0.9583333333333333, | |
| "mrr_at_100": 0.9583333333333333, | |
| "naucs_at_1_max": 0.7380368814192346, | |
| "naucs_at_1_std": 0.03408029878618052, | |
| "naucs_at_1_diff1": 0.9142156862745094, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| } | |
| } | |
| } |