Visual Document Retrieval
PEFT
Safetensors
ColPali
vidore
multimodal_embedding
multilingual_embedding
Text-to-Visual Document (T→VD) retrieval
zpn commited on
Commit
7728fdd
·
verified ·
1 Parent(s): 496c530

Update results.json

Browse files
Files changed (1) hide show
  1. results.json +1100 -1106
results.json CHANGED
@@ -1,1110 +1,1104 @@
1
  {
2
- "metadata": {
3
- "timestamp": "2025-03-31T23:54:56.140130",
4
- "vidore_benchmark_version": "5.0.1.dev6+g9e0da63"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  },
6
- "metrics": {
7
- "vidore/arxivqa_test_subsampled": {
8
- "ndcg_at_1": 0.83,
9
- "ndcg_at_3": 0.87533,
10
- "ndcg_at_5": 0.88678,
11
- "ndcg_at_10": 0.89447,
12
- "ndcg_at_20": 0.89988,
13
- "ndcg_at_50": 0.90114,
14
- "ndcg_at_100": 0.90247,
15
- "map_at_1": 0.83,
16
- "map_at_3": 0.86467,
17
- "map_at_5": 0.87097,
18
- "map_at_10": 0.87409,
19
- "map_at_20": 0.87551,
20
- "map_at_50": 0.87574,
21
- "map_at_100": 0.87587,
22
- "recall_at_1": 0.83,
23
- "recall_at_3": 0.906,
24
- "recall_at_5": 0.934,
25
- "recall_at_10": 0.958,
26
- "recall_at_20": 0.98,
27
- "recall_at_50": 0.986,
28
- "recall_at_100": 0.994,
29
- "precision_at_1": 0.83,
30
- "precision_at_3": 0.302,
31
- "precision_at_5": 0.1868,
32
- "precision_at_10": 0.0958,
33
- "precision_at_20": 0.049,
34
- "precision_at_50": 0.01972,
35
- "precision_at_100": 0.00994,
36
- "mrr_at_1": 0.832,
37
- "mrr_at_3": 0.8663333333333333,
38
- "mrr_at_5": 0.8716333333333333,
39
- "mrr_at_10": 0.8752095238095239,
40
- "mrr_at_20": 0.8764481089945486,
41
- "mrr_at_50": 0.8769236116400513,
42
- "mrr_at_100": 0.8770087474728994,
43
- "naucs_at_1_max": 0.7799094444766933,
44
- "naucs_at_1_std": -0.08580135833284933,
45
- "naucs_at_1_diff1": 0.9595692807801705,
46
- "naucs_at_3_max": 0.7955579394878525,
47
- "naucs_at_3_std": -0.11155809841667234,
48
- "naucs_at_3_diff1": 0.9224725351133368,
49
- "naucs_at_5_max": 0.7705061822708856,
50
- "naucs_at_5_std": -0.25026172084995735,
51
- "naucs_at_5_diff1": 0.9217666864725705,
52
- "naucs_at_10_max": 0.7739540260548654,
53
- "naucs_at_10_std": -0.21324085189631833,
54
- "naucs_at_10_diff1": 0.9221910986616823,
55
- "naucs_at_20_max": 0.7708216619981358,
56
- "naucs_at_20_std": -0.0996732026143794,
57
- "naucs_at_20_diff1": 0.8905228758169942,
58
- "naucs_at_50_max": 0.816593303988261,
59
- "naucs_at_50_std": -0.19987995198079136,
60
- "naucs_at_50_diff1": 0.8832866479925231,
61
- "naucs_at_100_max": 0.9564270152505304,
62
- "naucs_at_100_std": 0.04310613134141329,
63
- "naucs_at_100_diff1": 0.9564270152505304
64
- },
65
- "vidore/docvqa_test_subsampled": {
66
- "ndcg_at_1": 0.5388,
67
- "ndcg_at_3": 0.59542,
68
- "ndcg_at_5": 0.61269,
69
- "ndcg_at_10": 0.62919,
70
- "ndcg_at_20": 0.64259,
71
- "ndcg_at_50": 0.64929,
72
- "ndcg_at_100": 0.65683,
73
- "map_at_1": 0.5388,
74
- "map_at_3": 0.5813,
75
- "map_at_5": 0.59084,
76
- "map_at_10": 0.59764,
77
- "map_at_20": 0.6013,
78
- "map_at_50": 0.60241,
79
- "map_at_100": 0.60307,
80
- "recall_at_1": 0.5388,
81
- "recall_at_3": 0.63636,
82
- "recall_at_5": 0.67849,
83
- "recall_at_10": 0.72949,
84
- "recall_at_20": 0.78271,
85
- "recall_at_50": 0.81596,
86
- "recall_at_100": 0.86253,
87
- "precision_at_1": 0.5388,
88
- "precision_at_3": 0.21212,
89
- "precision_at_5": 0.1357,
90
- "precision_at_10": 0.07295,
91
- "precision_at_20": 0.03914,
92
- "precision_at_50": 0.01632,
93
- "precision_at_100": 0.00863,
94
- "mrr_at_1": 0.5432372505543237,
95
- "mrr_at_3": 0.5824094604582412,
96
- "mrr_at_5": 0.5903917220990392,
97
- "mrr_at_10": 0.5980097138633723,
98
- "mrr_at_20": 0.601820515422298,
99
- "mrr_at_50": 0.6029928998761527,
100
- "mrr_at_100": 0.6035667141572244,
101
- "naucs_at_1_max": 0.8430735352775856,
102
- "naucs_at_1_std": 0.5939128313043085,
103
- "naucs_at_1_diff1": 0.9018027917306838,
104
- "naucs_at_3_max": 0.8808230124613547,
105
- "naucs_at_3_std": 0.712681445860455,
106
- "naucs_at_3_diff1": 0.8646220134947054,
107
- "naucs_at_5_max": 0.8912698205476521,
108
- "naucs_at_5_std": 0.7525159249929689,
109
- "naucs_at_5_diff1": 0.8414611325660849,
110
- "naucs_at_10_max": 0.8852509111743405,
111
- "naucs_at_10_std": 0.817204422380503,
112
- "naucs_at_10_diff1": 0.8264628195305581,
113
- "naucs_at_20_max": 0.8512017211379178,
114
- "naucs_at_20_std": 0.8215729110215341,
115
- "naucs_at_20_diff1": 0.7992889048812711,
116
- "naucs_at_50_max": 0.8402551716801772,
117
- "naucs_at_50_std": 0.8733758748271155,
118
- "naucs_at_50_diff1": 0.7960833079573262,
119
- "naucs_at_100_max": 0.838262046357413,
120
- "naucs_at_100_std": 0.904184740422419,
121
- "naucs_at_100_diff1": 0.7996865824491111
122
- },
123
- "vidore/infovqa_test_subsampled": {
124
- "ndcg_at_1": 0.90081,
125
- "ndcg_at_3": 0.92934,
126
- "ndcg_at_5": 0.93352,
127
- "ndcg_at_10": 0.93669,
128
- "ndcg_at_20": 0.93871,
129
- "ndcg_at_50": 0.9403,
130
- "ndcg_at_100": 0.94099,
131
- "map_at_1": 0.90081,
132
- "map_at_3": 0.9224,
133
- "map_at_5": 0.92473,
134
- "map_at_10": 0.92598,
135
- "map_at_20": 0.92652,
136
- "map_at_50": 0.92677,
137
- "map_at_100": 0.92684,
138
- "recall_at_1": 0.90081,
139
- "recall_at_3": 0.94939,
140
- "recall_at_5": 0.95951,
141
- "recall_at_10": 0.96964,
142
- "recall_at_20": 0.97773,
143
- "recall_at_50": 0.98583,
144
- "recall_at_100": 0.98988,
145
- "precision_at_1": 0.90081,
146
- "precision_at_3": 0.31646,
147
- "precision_at_5": 0.1919,
148
- "precision_at_10": 0.09696,
149
- "precision_at_20": 0.04889,
150
- "precision_at_50": 0.01972,
151
- "precision_at_100": 0.0099,
152
- "mrr_at_1": 0.9008097165991903,
153
- "mrr_at_3": 0.9210526315789472,
154
- "mrr_at_5": 0.9248987854251012,
155
- "mrr_at_10": 0.9258040935672512,
156
- "mrr_at_20": 0.926239353314252,
157
- "mrr_at_50": 0.9265337558342984,
158
- "mrr_at_100": 0.9266429029872886,
159
- "naucs_at_1_max": 0.666687407456454,
160
- "naucs_at_1_std": 0.04348917328850024,
161
- "naucs_at_1_diff1": 0.9460185154989313,
162
- "naucs_at_3_max": 0.887692938235415,
163
- "naucs_at_3_std": 0.3724699632820944,
164
- "naucs_at_3_diff1": 0.9582075346992748,
165
- "naucs_at_5_max": 0.9121353932948266,
166
- "naucs_at_5_std": 0.46274982003896187,
167
- "naucs_at_5_diff1": 0.9608195637805694,
168
- "naucs_at_10_max": 0.9553640600031791,
169
- "naucs_at_10_std": 0.6527520622928011,
170
- "naucs_at_10_diff1": 0.9564661819784096,
171
- "naucs_at_20_max": 0.9643814216187027,
172
- "naucs_at_20_std": 0.8342696288850537,
173
- "naucs_at_20_diff1": 0.9643814216187027,
174
- "naucs_at_50_max": 1.0,
175
- "naucs_at_50_std": 0.960323608891707,
176
- "naucs_at_50_diff1": 0.9626852988386237,
177
- "naucs_at_100_max": 1.0,
178
- "naucs_at_100_std": 1.0,
179
- "naucs_at_100_diff1": 0.9477594183740937
180
- },
181
- "vidore/tabfquad_test_subsampled": {
182
- "ndcg_at_1": 0.925,
183
- "ndcg_at_3": 0.95561,
184
- "ndcg_at_5": 0.95991,
185
- "ndcg_at_10": 0.96464,
186
- "ndcg_at_20": 0.96464,
187
- "ndcg_at_50": 0.96464,
188
- "ndcg_at_100": 0.96464,
189
- "map_at_1": 0.925,
190
- "map_at_3": 0.94881,
191
- "map_at_5": 0.95113,
192
- "map_at_10": 0.95314,
193
- "map_at_20": 0.95314,
194
- "map_at_50": 0.95314,
195
- "map_at_100": 0.95314,
196
- "recall_at_1": 0.925,
197
- "recall_at_3": 0.975,
198
- "recall_at_5": 0.98571,
199
- "recall_at_10": 1.0,
200
- "recall_at_20": 1.0,
201
- "recall_at_50": 1.0,
202
- "recall_at_100": 1.0,
203
- "precision_at_1": 0.925,
204
- "precision_at_3": 0.325,
205
- "precision_at_5": 0.19714,
206
- "precision_at_10": 0.1,
207
- "precision_at_20": 0.05,
208
- "precision_at_50": 0.02,
209
- "precision_at_100": 0.01,
210
- "mrr_at_1": 0.9214285714285714,
211
- "mrr_at_3": 0.9464285714285714,
212
- "mrr_at_5": 0.9487499999999999,
213
- "mrr_at_10": 0.9507624716553289,
214
- "mrr_at_20": 0.9507624716553289,
215
- "mrr_at_50": 0.9507624716553289,
216
- "mrr_at_100": 0.9507624716553289,
217
- "naucs_at_1_max": 0.5344804588502072,
218
- "naucs_at_1_std": 0.26817393624116476,
219
- "naucs_at_1_diff1": 0.9383086567960518,
220
- "naucs_at_3_max": 0.9416433239962675,
221
- "naucs_at_3_std": 0.8989595838335351,
222
- "naucs_at_3_diff1": 0.98132586367881,
223
- "naucs_at_5_max": 0.967320261437913,
224
- "naucs_at_5_std": 0.8558590102707857,
225
- "naucs_at_5_diff1": 0.967320261437913,
226
- "naucs_at_10_max": 1.0,
227
- "naucs_at_10_std": 1.0,
228
- "naucs_at_10_diff1": 1.0,
229
- "naucs_at_20_max": 1.0,
230
- "naucs_at_20_std": 1.0,
231
- "naucs_at_20_diff1": 1.0,
232
- "naucs_at_50_max": 1.0,
233
- "naucs_at_50_std": 1.0,
234
- "naucs_at_50_diff1": 1.0,
235
- "naucs_at_100_max": 1.0,
236
- "naucs_at_100_std": 1.0,
237
- "naucs_at_100_diff1": 1.0
238
- },
239
- "vidore/tatdqa_test": {
240
- "ndcg_at_1": 0.70109,
241
- "ndcg_at_3": 0.79187,
242
- "ndcg_at_5": 0.81111,
243
- "ndcg_at_10": 0.82342,
244
- "ndcg_at_20": 0.8274,
245
- "ndcg_at_50": 0.83026,
246
- "ndcg_at_100": 0.83293,
247
- "map_at_1": 0.70109,
248
- "map_at_3": 0.77005,
249
- "map_at_5": 0.78071,
250
- "map_at_10": 0.78598,
251
- "map_at_20": 0.78706,
252
- "map_at_50": 0.78751,
253
- "map_at_100": 0.78775,
254
- "recall_at_1": 0.70109,
255
- "recall_at_3": 0.8548,
256
- "recall_at_5": 0.90158,
257
- "recall_at_10": 0.93864,
258
- "recall_at_20": 0.95443,
259
- "recall_at_50": 0.96902,
260
- "recall_at_100": 0.98542,
261
- "precision_at_1": 0.70109,
262
- "precision_at_3": 0.28493,
263
- "precision_at_5": 0.18032,
264
- "precision_at_10": 0.09386,
265
- "precision_at_20": 0.04772,
266
- "precision_at_50": 0.01938,
267
- "precision_at_100": 0.00985,
268
- "mrr_at_1": 0.7023086269744836,
269
- "mrr_at_3": 0.7714661806399354,
270
- "mrr_at_5": 0.782978938841637,
271
- "mrr_at_10": 0.7879385330478899,
272
- "mrr_at_20": 0.7889867733618092,
273
- "mrr_at_50": 0.7893770561483674,
274
- "mrr_at_100": 0.7896057919808999,
275
- "naucs_at_1_max": 0.25184895001768326,
276
- "naucs_at_1_std": 0.037017257538968965,
277
- "naucs_at_1_diff1": 0.8508843501202716,
278
- "naucs_at_3_max": 0.28836101906742934,
279
- "naucs_at_3_std": 0.0895587034833779,
280
- "naucs_at_3_diff1": 0.7677154118662606,
281
- "naucs_at_5_max": 0.33141107758129335,
282
- "naucs_at_5_std": 0.1402343443047445,
283
- "naucs_at_5_diff1": 0.726113129605695,
284
- "naucs_at_10_max": 0.3753484131980775,
285
- "naucs_at_10_std": 0.21181536222478634,
286
- "naucs_at_10_diff1": 0.7110698774619854,
287
- "naucs_at_20_max": 0.398127644970406,
288
- "naucs_at_20_std": 0.2900002928945751,
289
- "naucs_at_20_diff1": 0.6944425419244158,
290
- "naucs_at_50_max": 0.4094042090297014,
291
- "naucs_at_50_std": 0.3282254760208566,
292
- "naucs_at_50_diff1": 0.665997592036148,
293
- "naucs_at_100_max": 0.2601633836764814,
294
- "naucs_at_100_std": 0.2532897060083396,
295
- "naucs_at_100_diff1": 0.6808982214080673
296
- },
297
- "vidore/shiftproject_test": {
298
- "ndcg_at_1": 0.84,
299
- "ndcg_at_3": 0.91809,
300
- "ndcg_at_5": 0.91809,
301
- "ndcg_at_10": 0.92499,
302
- "ndcg_at_20": 0.92499,
303
- "ndcg_at_50": 0.92499,
304
- "ndcg_at_100": 0.92673,
305
- "map_at_1": 0.84,
306
- "map_at_3": 0.9,
307
- "map_at_5": 0.9,
308
- "map_at_10": 0.9031,
309
- "map_at_20": 0.9031,
310
- "map_at_50": 0.9031,
311
- "map_at_100": 0.90328,
312
- "recall_at_1": 0.84,
313
- "recall_at_3": 0.97,
314
- "recall_at_5": 0.97,
315
- "recall_at_10": 0.99,
316
- "recall_at_20": 0.99,
317
- "recall_at_50": 0.99,
318
- "recall_at_100": 1.0,
319
- "precision_at_1": 0.84,
320
- "precision_at_3": 0.32333,
321
- "precision_at_5": 0.194,
322
- "precision_at_10": 0.099,
323
- "precision_at_20": 0.0495,
324
- "precision_at_50": 0.0198,
325
- "precision_at_100": 0.01,
326
- "mrr_at_1": 0.84,
327
- "mrr_at_3": 0.8983333333333333,
328
- "mrr_at_5": 0.8983333333333333,
329
- "mrr_at_10": 0.9014285714285714,
330
- "mrr_at_20": 0.9014285714285714,
331
- "mrr_at_50": 0.9014285714285714,
332
- "mrr_at_100": 0.9016208791208791,
333
- "naucs_at_1_max": 0.1685076092292596,
334
- "naucs_at_1_std": -0.2145925380461453,
335
- "naucs_at_1_diff1": 0.7751595483554242,
336
- "naucs_at_3_max": 0.6498599439775861,
337
- "naucs_at_3_std": 0.36834733893557625,
338
- "naucs_at_3_diff1": 0.664021164021167,
339
- "naucs_at_5_max": 0.6498599439775937,
340
- "naucs_at_5_std": 0.36834733893557176,
341
- "naucs_at_5_diff1": 0.6640211640211615,
342
- "naucs_at_10_max": 0.7222222222222276,
343
- "naucs_at_10_std": 0.5541549953314738,
344
- "naucs_at_10_diff1": 0.8692810457516413,
345
- "naucs_at_20_max": 0.7222222222222276,
346
- "naucs_at_20_std": 0.5541549953314738,
347
- "naucs_at_20_diff1": 0.8692810457516413,
348
- "naucs_at_50_max": 0.7222222222222041,
349
- "naucs_at_50_std": 0.554154995331464,
350
- "naucs_at_50_diff1": 0.8692810457516374,
351
- "naucs_at_100_max": null,
352
- "naucs_at_100_std": null,
353
- "naucs_at_100_diff1": null
354
- },
355
- "vidore/syntheticDocQA_artificial_intelligence_test": {
356
- "ndcg_at_1": 0.98,
357
- "ndcg_at_3": 0.99262,
358
- "ndcg_at_5": 0.99262,
359
- "ndcg_at_10": 0.99262,
360
- "ndcg_at_20": 0.99262,
361
- "ndcg_at_50": 0.99262,
362
- "ndcg_at_100": 0.99262,
363
- "map_at_1": 0.98,
364
- "map_at_3": 0.99,
365
- "map_at_5": 0.99,
366
- "map_at_10": 0.99,
367
- "map_at_20": 0.99,
368
- "map_at_50": 0.99,
369
- "map_at_100": 0.99,
370
- "recall_at_1": 0.98,
371
- "recall_at_3": 1.0,
372
- "recall_at_5": 1.0,
373
- "recall_at_10": 1.0,
374
- "recall_at_20": 1.0,
375
- "recall_at_50": 1.0,
376
- "recall_at_100": 1.0,
377
- "precision_at_1": 0.98,
378
- "precision_at_3": 0.33333,
379
- "precision_at_5": 0.2,
380
- "precision_at_10": 0.1,
381
- "precision_at_20": 0.05,
382
- "precision_at_50": 0.02,
383
- "precision_at_100": 0.01,
384
- "mrr_at_1": 0.99,
385
- "mrr_at_3": 0.995,
386
- "mrr_at_5": 0.995,
387
- "mrr_at_10": 0.995,
388
- "mrr_at_20": 0.995,
389
- "mrr_at_50": 0.995,
390
- "mrr_at_100": 0.995,
391
- "naucs_at_1_max": 0.540149393090569,
392
- "naucs_at_1_std": -0.6909430438842186,
393
- "naucs_at_1_diff1": 1.0,
394
- "naucs_at_3_max": 1.0,
395
- "naucs_at_3_std": 1.0,
396
- "naucs_at_3_diff1": 1.0,
397
- "naucs_at_5_max": 1.0,
398
- "naucs_at_5_std": 1.0,
399
- "naucs_at_5_diff1": 1.0,
400
- "naucs_at_10_max": 1.0,
401
- "naucs_at_10_std": 1.0,
402
- "naucs_at_10_diff1": 1.0,
403
- "naucs_at_20_max": 1.0,
404
- "naucs_at_20_std": 1.0,
405
- "naucs_at_20_diff1": 1.0,
406
- "naucs_at_50_max": null,
407
- "naucs_at_50_std": null,
408
- "naucs_at_50_diff1": null,
409
- "naucs_at_100_max": null,
410
- "naucs_at_100_std": null,
411
- "naucs_at_100_diff1": null
412
- },
413
- "vidore/syntheticDocQA_energy_test": {
414
- "ndcg_at_1": 0.95,
415
- "ndcg_at_3": 0.95631,
416
- "ndcg_at_5": 0.96492,
417
- "ndcg_at_10": 0.96781,
418
- "ndcg_at_20": 0.96781,
419
- "ndcg_at_50": 0.96976,
420
- "ndcg_at_100": 0.96976,
421
- "map_at_1": 0.95,
422
- "map_at_3": 0.955,
423
- "map_at_5": 0.96,
424
- "map_at_10": 0.961,
425
- "map_at_20": 0.961,
426
- "map_at_50": 0.96129,
427
- "map_at_100": 0.96129,
428
- "recall_at_1": 0.95,
429
- "recall_at_3": 0.96,
430
- "recall_at_5": 0.98,
431
- "recall_at_10": 0.99,
432
- "recall_at_20": 0.99,
433
- "recall_at_50": 1.0,
434
- "recall_at_100": 1.0,
435
- "precision_at_1": 0.95,
436
- "precision_at_3": 0.32,
437
- "precision_at_5": 0.196,
438
- "precision_at_10": 0.099,
439
- "precision_at_20": 0.0495,
440
- "precision_at_50": 0.02,
441
- "precision_at_100": 0.01,
442
- "mrr_at_1": 0.95,
443
- "mrr_at_3": 0.955,
444
- "mrr_at_5": 0.96,
445
- "mrr_at_10": 0.961,
446
- "mrr_at_20": 0.961,
447
- "mrr_at_50": 0.9613030303030302,
448
- "mrr_at_100": 0.9613030303030302,
449
- "naucs_at_1_max": 0.32362278244631343,
450
- "naucs_at_1_std": -0.7713352007469629,
451
- "naucs_at_1_diff1": 1.0,
452
- "naucs_at_3_max": 0.8395191409897237,
453
- "naucs_at_3_std": -0.6762371615312763,
454
- "naucs_at_3_diff1": 1.0,
455
- "naucs_at_5_max": 0.6790382819794609,
456
- "naucs_at_5_std": -1.445845004668519,
457
- "naucs_at_5_diff1": 1.0,
458
- "naucs_at_10_max": 1.0,
459
- "naucs_at_10_std": -1.1517273576097316,
460
- "naucs_at_10_diff1": 1.0,
461
- "naucs_at_20_max": 1.0,
462
- "naucs_at_20_std": -1.1517273576097316,
463
- "naucs_at_20_diff1": 1.0,
464
- "naucs_at_50_max": null,
465
- "naucs_at_50_std": null,
466
- "naucs_at_50_diff1": null,
467
- "naucs_at_100_max": null,
468
- "naucs_at_100_std": null,
469
- "naucs_at_100_diff1": null
470
- },
471
- "vidore/syntheticDocQA_government_reports_test": {
472
- "ndcg_at_1": 0.91,
473
- "ndcg_at_3": 0.94655,
474
- "ndcg_at_5": 0.95085,
475
- "ndcg_at_10": 0.95743,
476
- "ndcg_at_20": 0.95743,
477
- "ndcg_at_50": 0.95743,
478
- "ndcg_at_100": 0.95743,
479
- "map_at_1": 0.91,
480
- "map_at_3": 0.93833,
481
- "map_at_5": 0.94083,
482
- "map_at_10": 0.94361,
483
- "map_at_20": 0.94361,
484
- "map_at_50": 0.94361,
485
- "map_at_100": 0.94361,
486
- "recall_at_1": 0.91,
487
- "recall_at_3": 0.97,
488
- "recall_at_5": 0.98,
489
- "recall_at_10": 1.0,
490
- "recall_at_20": 1.0,
491
- "recall_at_50": 1.0,
492
- "recall_at_100": 1.0,
493
- "precision_at_1": 0.91,
494
- "precision_at_3": 0.32333,
495
- "precision_at_5": 0.196,
496
- "precision_at_10": 0.1,
497
- "precision_at_20": 0.05,
498
- "precision_at_50": 0.02,
499
- "precision_at_100": 0.01,
500
- "mrr_at_1": 0.92,
501
- "mrr_at_3": 0.9433333333333332,
502
- "mrr_at_5": 0.9483333333333333,
503
- "mrr_at_10": 0.9494444444444444,
504
- "mrr_at_20": 0.9494444444444444,
505
- "mrr_at_50": 0.9494444444444444,
506
- "mrr_at_100": 0.9494444444444444,
507
- "naucs_at_1_max": 0.6774561676522453,
508
- "naucs_at_1_std": 0.14721444133208894,
509
- "naucs_at_1_diff1": 0.9564270152505436,
510
- "naucs_at_3_max": 0.8513849984438244,
511
- "naucs_at_3_std": -0.24929971988795643,
512
- "naucs_at_3_diff1": 0.9564270152505466,
513
- "naucs_at_5_max": 1.0,
514
- "naucs_at_5_std": -0.43534080298785716,
515
- "naucs_at_5_diff1": 0.9346405228758136,
516
- "naucs_at_10_max": 1.0,
517
- "naucs_at_10_std": 1.0,
518
- "naucs_at_10_diff1": 1.0,
519
- "naucs_at_20_max": 1.0,
520
- "naucs_at_20_std": 1.0,
521
- "naucs_at_20_diff1": 1.0,
522
- "naucs_at_50_max": null,
523
- "naucs_at_50_std": null,
524
- "naucs_at_50_diff1": null,
525
- "naucs_at_100_max": null,
526
- "naucs_at_100_std": null,
527
- "naucs_at_100_diff1": null
528
- },
529
- "vidore/syntheticDocQA_healthcare_industry_test": {
530
- "ndcg_at_1": 0.98,
531
- "ndcg_at_3": 0.99262,
532
- "ndcg_at_5": 0.99262,
533
- "ndcg_at_10": 0.99262,
534
- "ndcg_at_20": 0.99262,
535
- "ndcg_at_50": 0.99262,
536
- "ndcg_at_100": 0.99262,
537
- "map_at_1": 0.98,
538
- "map_at_3": 0.99,
539
- "map_at_5": 0.99,
540
- "map_at_10": 0.99,
541
- "map_at_20": 0.99,
542
- "map_at_50": 0.99,
543
- "map_at_100": 0.99,
544
- "recall_at_1": 0.98,
545
- "recall_at_3": 1.0,
546
- "recall_at_5": 1.0,
547
- "recall_at_10": 1.0,
548
- "recall_at_20": 1.0,
549
- "recall_at_50": 1.0,
550
- "recall_at_100": 1.0,
551
- "precision_at_1": 0.98,
552
- "precision_at_3": 0.33333,
553
- "precision_at_5": 0.2,
554
- "precision_at_10": 0.1,
555
- "precision_at_20": 0.05,
556
- "precision_at_50": 0.02,
557
- "precision_at_100": 0.01,
558
- "mrr_at_1": 0.98,
559
- "mrr_at_3": 0.99,
560
- "mrr_at_5": 0.99,
561
- "mrr_at_10": 0.99,
562
- "mrr_at_20": 0.99,
563
- "mrr_at_50": 0.99,
564
- "mrr_at_100": 0.99,
565
- "naucs_at_1_max": 0.7222222222222248,
566
- "naucs_at_1_std": -0.5634920634920563,
567
- "naucs_at_1_diff1": 1.0,
568
- "naucs_at_3_max": 1.0,
569
- "naucs_at_3_std": 1.0,
570
- "naucs_at_3_diff1": 1.0,
571
- "naucs_at_5_max": 1.0,
572
- "naucs_at_5_std": 1.0,
573
- "naucs_at_5_diff1": 1.0,
574
- "naucs_at_10_max": 1.0,
575
- "naucs_at_10_std": 1.0,
576
- "naucs_at_10_diff1": 1.0,
577
- "naucs_at_20_max": 1.0,
578
- "naucs_at_20_std": 1.0,
579
- "naucs_at_20_diff1": 1.0,
580
- "naucs_at_50_max": null,
581
- "naucs_at_50_std": null,
582
- "naucs_at_50_diff1": null,
583
- "naucs_at_100_max": null,
584
- "naucs_at_100_std": null,
585
- "naucs_at_100_diff1": null
586
- },
587
- "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": {
588
- "ndcg_at_1": 0.66875,
589
- "ndcg_at_3": 0.64833,
590
- "ndcg_at_5": 0.66136,
591
- "ndcg_at_10": 0.7002,
592
- "ndcg_at_20": 0.72343,
593
- "ndcg_at_50": 0.74221,
594
- "ndcg_at_100": 0.7504,
595
- "map_at_1": 0.41001,
596
- "map_at_3": 0.53585,
597
- "map_at_5": 0.57453,
598
- "map_at_10": 0.61456,
599
- "map_at_20": 0.63039,
600
- "map_at_50": 0.63734,
601
- "map_at_100": 0.63926,
602
- "recall_at_1": 0.41001,
603
- "recall_at_3": 0.59207,
604
- "recall_at_5": 0.66951,
605
- "recall_at_10": 0.7842,
606
- "recall_at_20": 0.84612,
607
- "recall_at_50": 0.90015,
608
- "recall_at_100": 0.92697,
609
- "precision_at_1": 0.66875,
610
- "precision_at_3": 0.39375,
611
- "precision_at_5": 0.29125,
612
- "precision_at_10": 0.18938,
613
- "precision_at_20": 0.11,
614
- "precision_at_50": 0.05087,
615
- "precision_at_100": 0.0275,
616
- "mrr_at_1": 0.65,
617
- "mrr_at_3": 0.7364583333333331,
618
- "mrr_at_5": 0.7480208333333331,
619
- "mrr_at_10": 0.7537127976190475,
620
- "mrr_at_20": 0.7549048402255638,
621
- "mrr_at_50": 0.755591370159045,
622
- "mrr_at_100": 0.755591370159045,
623
- "naucs_at_1_max": 0.3941383204398901,
624
- "naucs_at_1_std": -0.03446929320007161,
625
- "naucs_at_1_diff1": 0.4503277010269023,
626
- "naucs_at_3_max": 0.029009734517562568,
627
- "naucs_at_3_std": 0.005402681474374929,
628
- "naucs_at_3_diff1": -0.048706505991403734,
629
- "naucs_at_5_max": -0.08207834225628742,
630
- "naucs_at_5_std": -0.0032984001708723115,
631
- "naucs_at_5_diff1": -0.17283336776461594,
632
- "naucs_at_10_max": -0.17977285443640584,
633
- "naucs_at_10_std": -0.04541684253923944,
634
- "naucs_at_10_diff1": -0.23295672062273604,
635
- "naucs_at_20_max": -0.170593905587224,
636
- "naucs_at_20_std": 0.015384406511413186,
637
- "naucs_at_20_diff1": -0.2551053459454954,
638
- "naucs_at_50_max": -0.22984435384481694,
639
- "naucs_at_50_std": 0.04443493398633732,
640
- "naucs_at_50_diff1": -0.27937793023113594,
641
- "naucs_at_100_max": -0.25957527630817795,
642
- "naucs_at_100_std": 0.057081217133354005,
643
- "naucs_at_100_diff1": -0.3033742727110607
644
- },
645
- "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": {
646
- "ndcg_at_1": 0.65517,
647
- "ndcg_at_3": 0.62816,
648
- "ndcg_at_5": 0.6159,
649
- "ndcg_at_10": 0.58912,
650
- "ndcg_at_20": 0.61706,
651
- "ndcg_at_50": 0.67765,
652
- "ndcg_at_100": 0.71471,
653
- "map_at_1": 0.08673,
654
- "map_at_3": 0.19894,
655
- "map_at_5": 0.26716,
656
- "map_at_10": 0.33292,
657
- "map_at_20": 0.40089,
658
- "map_at_50": 0.45667,
659
- "map_at_100": 0.4844,
660
- "recall_at_1": 0.08673,
661
- "recall_at_3": 0.24596,
662
- "recall_at_5": 0.34869,
663
- "recall_at_10": 0.4683,
664
- "recall_at_20": 0.62193,
665
- "recall_at_50": 0.80907,
666
- "recall_at_100": 0.92119,
667
- "precision_at_1": 0.65517,
668
- "precision_at_3": 0.58621,
669
- "precision_at_5": 0.55172,
670
- "precision_at_10": 0.42931,
671
- "precision_at_20": 0.32672,
672
- "precision_at_50": 0.19759,
673
- "precision_at_100": 0.1281,
674
- "mrr_at_1": 0.6896551724137931,
675
- "mrr_at_3": 0.7902298850574713,
676
- "mrr_at_5": 0.7971264367816091,
677
- "mrr_at_10": 0.8015051997810618,
678
- "mrr_at_20": 0.8015051997810618,
679
- "mrr_at_50": 0.8023262178434591,
680
- "mrr_at_100": 0.8023262178434591,
681
- "naucs_at_1_max": -0.046307070115366736,
682
- "naucs_at_1_std": 0.15790688174002224,
683
- "naucs_at_1_diff1": -0.14987305341556462,
684
- "naucs_at_3_max": 0.11671339502888234,
685
- "naucs_at_3_std": 0.11834362034275059,
686
- "naucs_at_3_diff1": -0.17762652230301346,
687
- "naucs_at_5_max": 0.10679449683484428,
688
- "naucs_at_5_std": 0.09453878709557821,
689
- "naucs_at_5_diff1": -0.1792193824994377,
690
- "naucs_at_10_max": 0.10651036963192122,
691
- "naucs_at_10_std": 0.207447480653671,
692
- "naucs_at_10_diff1": -0.23752907816911517,
693
- "naucs_at_20_max": 0.05980777985340397,
694
- "naucs_at_20_std": 0.23365933479100473,
695
- "naucs_at_20_diff1": -0.2025898492844735,
696
- "naucs_at_50_max": -0.012623885278852142,
697
- "naucs_at_50_std": 0.2463485456131626,
698
- "naucs_at_50_diff1": -0.18909262376408734,
699
- "naucs_at_100_max": -0.08872391317584416,
700
- "naucs_at_100_std": 0.17948141514389135,
701
- "naucs_at_100_diff1": -0.186930857132979
702
- },
703
- "vidore/synthetic_rse_restaurant_filtered_v1.0": {
704
- "ndcg_at_1": 0.52632,
705
- "ndcg_at_3": 0.52504,
706
- "ndcg_at_5": 0.57289,
707
- "ndcg_at_10": 0.62003,
708
- "ndcg_at_20": 0.65505,
709
- "ndcg_at_50": 0.68979,
710
- "ndcg_at_100": 0.70204,
711
- "map_at_1": 0.29474,
712
- "map_at_3": 0.40396,
713
- "map_at_5": 0.45972,
714
- "map_at_10": 0.50866,
715
- "map_at_20": 0.5276,
716
- "map_at_50": 0.54534,
717
- "map_at_100": 0.55161,
718
- "recall_at_1": 0.29474,
719
- "recall_at_3": 0.48191,
720
- "recall_at_5": 0.61543,
721
- "recall_at_10": 0.75002,
722
- "recall_at_20": 0.85537,
723
- "recall_at_50": 0.94841,
724
- "recall_at_100": 0.98246,
725
- "precision_at_1": 0.52632,
726
- "precision_at_3": 0.34503,
727
- "precision_at_5": 0.28421,
728
- "precision_at_10": 0.19649,
729
- "precision_at_20": 0.12281,
730
- "precision_at_50": 0.06912,
731
- "precision_at_100": 0.03877,
732
- "mrr_at_1": 0.543859649122807,
733
- "mrr_at_3": 0.6403508771929824,
734
- "mrr_at_5": 0.6526315789473685,
735
- "mrr_at_10": 0.6602548036758563,
736
- "mrr_at_20": 0.6651739829371409,
737
- "mrr_at_50": 0.6655248601301235,
738
- "mrr_at_100": 0.6655248601301235,
739
- "naucs_at_1_max": 0.07429395157545386,
740
- "naucs_at_1_std": 0.03015644215746205,
741
- "naucs_at_1_diff1": 0.029188519373247488,
742
- "naucs_at_3_max": -0.14556834354246306,
743
- "naucs_at_3_std": -0.03551874034674307,
744
- "naucs_at_3_diff1": -0.03426575331185958,
745
- "naucs_at_5_max": -0.132667608690031,
746
- "naucs_at_5_std": 0.046883073270420496,
747
- "naucs_at_5_diff1": 0.020789980948042533,
748
- "naucs_at_10_max": -0.15350074281217774,
749
- "naucs_at_10_std": 0.021602996730365333,
750
- "naucs_at_10_diff1": 0.10610855445931551,
751
- "naucs_at_20_max": -0.2870595843942612,
752
- "naucs_at_20_std": -0.053200969005437526,
753
- "naucs_at_20_diff1": 0.009614869287472011,
754
- "naucs_at_50_max": -0.3747840510110352,
755
- "naucs_at_50_std": -0.1315278407645466,
756
- "naucs_at_50_diff1": -0.08682413153737878,
757
- "naucs_at_100_max": -0.3797677839881161,
758
- "naucs_at_100_std": -0.11609328262961503,
759
- "naucs_at_100_diff1": -0.12296867865157128
760
- },
761
- "vidore/synthetic_axa_filtered_v1.0": {
762
- "ndcg_at_1": 0.61111,
763
- "ndcg_at_3": 0.6462,
764
- "ndcg_at_5": 0.68292,
765
- "ndcg_at_10": 0.70017,
766
- "ndcg_at_20": 0.72201,
767
- "ndcg_at_50": 0.7613,
768
- "ndcg_at_100": 0.7658,
769
- "map_at_1": 0.29544,
770
- "map_at_3": 0.41914,
771
- "map_at_5": 0.49958,
772
- "map_at_10": 0.57784,
773
- "map_at_20": 0.61104,
774
- "map_at_50": 0.63144,
775
- "map_at_100": 0.63315,
776
- "recall_at_1": 0.29544,
777
- "recall_at_3": 0.51973,
778
- "recall_at_5": 0.66378,
779
- "recall_at_10": 0.78472,
780
- "recall_at_20": 0.85945,
781
- "recall_at_50": 0.97412,
782
- "recall_at_100": 0.98765,
783
- "precision_at_1": 0.61111,
784
- "precision_at_3": 0.46296,
785
- "precision_at_5": 0.42222,
786
- "precision_at_10": 0.31111,
787
- "precision_at_20": 0.19167,
788
- "precision_at_50": 0.09111,
789
- "precision_at_100": 0.04667,
790
- "mrr_at_1": 0.5555555555555556,
791
- "mrr_at_3": 0.6759259259259259,
792
- "mrr_at_5": 0.7037037037037037,
793
- "mrr_at_10": 0.7037037037037037,
794
- "mrr_at_20": 0.7037037037037037,
795
- "mrr_at_50": 0.7060185185185185,
796
- "mrr_at_100": 0.7060185185185185,
797
- "naucs_at_1_max": 0.06370640145678692,
798
- "naucs_at_1_std": 0.42153041887902504,
799
- "naucs_at_1_diff1": 0.35531111328419196,
800
- "naucs_at_3_max": -0.49137089517804067,
801
- "naucs_at_3_std": -0.41095639069482337,
802
- "naucs_at_3_diff1": 0.01707919337550571,
803
- "naucs_at_5_max": -0.6120658234932744,
804
- "naucs_at_5_std": -0.474389941591449,
805
- "naucs_at_5_diff1": -0.165061956148533,
806
- "naucs_at_10_max": -0.6883907404624792,
807
- "naucs_at_10_std": -0.45513671453258775,
808
- "naucs_at_10_diff1": -0.42316346500786917,
809
- "naucs_at_20_max": -0.733957515102311,
810
- "naucs_at_20_std": -0.4723325528538845,
811
- "naucs_at_20_diff1": -0.4807537601912433,
812
- "naucs_at_50_max": -0.7013881037720148,
813
- "naucs_at_50_std": -0.45324759826825234,
814
- "naucs_at_50_diff1": -0.4882986634795409,
815
- "naucs_at_100_max": -0.6924248438890106,
816
- "naucs_at_100_std": -0.4400487565870318,
817
- "naucs_at_100_diff1": -0.4661845036852439
818
- },
819
- "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": {
820
- "ndcg_at_1": 0.54825,
821
- "ndcg_at_3": 0.53008,
822
- "ndcg_at_5": 0.56705,
823
- "ndcg_at_10": 0.6257,
824
- "ndcg_at_20": 0.6625,
825
- "ndcg_at_50": 0.69353,
826
- "ndcg_at_100": 0.70465,
827
- "map_at_1": 0.29816,
828
- "map_at_3": 0.4016,
829
- "map_at_5": 0.4527,
830
- "map_at_10": 0.50738,
831
- "map_at_20": 0.52962,
832
- "map_at_50": 0.5466,
833
- "map_at_100": 0.55224,
834
- "recall_at_1": 0.29816,
835
- "recall_at_3": 0.48014,
836
- "recall_at_5": 0.59266,
837
- "recall_at_10": 0.76588,
838
- "recall_at_20": 0.87318,
839
- "recall_at_50": 0.95235,
840
- "recall_at_100": 0.98465,
841
- "precision_at_1": 0.54825,
842
- "precision_at_3": 0.34795,
843
- "precision_at_5": 0.28158,
844
- "precision_at_10": 0.19956,
845
- "precision_at_20": 0.12654,
846
- "precision_at_50": 0.06991,
847
- "precision_at_100": 0.03877,
848
- "mrr_at_1": 0.5394736842105263,
849
- "mrr_at_3": 0.6381578947368421,
850
- "mrr_at_5": 0.6528508771929824,
851
- "mrr_at_10": 0.666278543581175,
852
- "mrr_at_20": 0.6693822460076329,
853
- "mrr_at_50": 0.6696454039023697,
854
- "mrr_at_100": 0.6697038834345334,
855
- "naucs_at_1_max": -0.10947777032499051,
856
- "naucs_at_1_std": -0.08674361998375714,
857
- "naucs_at_1_diff1": 0.057249370353071344,
858
- "naucs_at_3_max": -0.06687224875668692,
859
- "naucs_at_3_std": 0.0484364629589946,
860
- "naucs_at_3_diff1": -0.05462426569089335,
861
- "naucs_at_5_max": -0.08403937224497868,
862
- "naucs_at_5_std": 0.07293419772827552,
863
- "naucs_at_5_diff1": -0.05958666027516894,
864
- "naucs_at_10_max": -0.10838130867204888,
865
- "naucs_at_10_std": 0.05587982504285361,
866
- "naucs_at_10_diff1": 0.002192750308197299,
867
- "naucs_at_20_max": -0.1628596976822229,
868
- "naucs_at_20_std": 0.06415536794747155,
869
- "naucs_at_20_diff1": -0.0901131230050924,
870
- "naucs_at_50_max": -0.2471444987505313,
871
- "naucs_at_50_std": 0.02178464235724526,
872
- "naucs_at_50_diff1": -0.16484319783599452,
873
- "naucs_at_100_max": -0.2476326734790779,
874
- "naucs_at_100_std": 0.04921366491713892,
875
- "naucs_at_100_diff1": -0.1925485969942229
876
- },
877
- "vidore/synthetic_axa_filtered_v1.0_multilingual": {
878
- "ndcg_at_1": 0.56944,
879
- "ndcg_at_3": 0.60114,
880
- "ndcg_at_5": 0.61308,
881
- "ndcg_at_10": 0.63301,
882
- "ndcg_at_20": 0.65962,
883
- "ndcg_at_50": 0.70087,
884
- "ndcg_at_100": 0.71794,
885
- "map_at_1": 0.26807,
886
- "map_at_3": 0.38274,
887
- "map_at_5": 0.44543,
888
- "map_at_10": 0.51783,
889
- "map_at_20": 0.55091,
890
- "map_at_50": 0.57108,
891
- "map_at_100": 0.57464,
892
- "recall_at_1": 0.26807,
893
- "recall_at_3": 0.47345,
894
- "recall_at_5": 0.56537,
895
- "recall_at_10": 0.69703,
896
- "recall_at_20": 0.78343,
897
- "recall_at_50": 0.90152,
898
- "recall_at_100": 0.97889,
899
- "precision_at_1": 0.56944,
900
- "precision_at_3": 0.43056,
901
- "precision_at_5": 0.38056,
902
- "precision_at_10": 0.27917,
903
- "precision_at_20": 0.17569,
904
- "precision_at_50": 0.08583,
905
- "precision_at_100": 0.04569,
906
- "mrr_at_1": 0.5,
907
- "mrr_at_3": 0.6481481481481483,
908
- "mrr_at_5": 0.6550925925925927,
909
- "mrr_at_10": 0.6566358024691359,
910
- "mrr_at_20": 0.6577041785375121,
911
- "mrr_at_50": 0.659896412080552,
912
- "mrr_at_100": 0.6607831288544433,
913
- "naucs_at_1_max": 0.16468725740231643,
914
- "naucs_at_1_std": 0.37919017176717934,
915
- "naucs_at_1_diff1": 0.30214037198755495,
916
- "naucs_at_3_max": -0.2419012234949494,
917
- "naucs_at_3_std": -0.11880659562104584,
918
- "naucs_at_3_diff1": -0.05983117053027332,
919
- "naucs_at_5_max": -0.3435266789444122,
920
- "naucs_at_5_std": -0.15122373891433766,
921
- "naucs_at_5_diff1": -0.14301810021483025,
922
- "naucs_at_10_max": -0.4049299119799346,
923
- "naucs_at_10_std": -0.11551373909975436,
924
- "naucs_at_10_diff1": -0.3087348082925975,
925
- "naucs_at_20_max": -0.4557080881366001,
926
- "naucs_at_20_std": -0.1316035057344777,
927
- "naucs_at_20_diff1": -0.3767740930238455,
928
- "naucs_at_50_max": -0.48221867587284106,
929
- "naucs_at_50_std": -0.16520884486012927,
930
- "naucs_at_50_diff1": -0.437539802211166,
931
- "naucs_at_100_max": -0.5041035001503106,
932
- "naucs_at_100_std": -0.19246332353391876,
933
- "naucs_at_100_diff1": -0.450877397124322
934
- },
935
- "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": {
936
- "ndcg_at_1": 0.62656,
937
- "ndcg_at_3": 0.62898,
938
- "ndcg_at_5": 0.64249,
939
- "ndcg_at_10": 0.67441,
940
- "ndcg_at_20": 0.69935,
941
- "ndcg_at_50": 0.72034,
942
- "ndcg_at_100": 0.7304,
943
- "map_at_1": 0.37553,
944
- "map_at_3": 0.51341,
945
- "map_at_5": 0.55134,
946
- "map_at_10": 0.58654,
947
- "map_at_20": 0.60275,
948
- "map_at_50": 0.61047,
949
- "map_at_100": 0.61272,
950
- "recall_at_1": 0.37553,
951
- "recall_at_3": 0.58317,
952
- "recall_at_5": 0.6651,
953
- "recall_at_10": 0.76267,
954
- "recall_at_20": 0.83047,
955
- "recall_at_50": 0.89402,
956
- "recall_at_100": 0.93005,
957
- "precision_at_1": 0.62656,
958
- "precision_at_3": 0.39115,
959
- "precision_at_5": 0.28812,
960
- "precision_at_10": 0.18453,
961
- "precision_at_20": 0.10859,
962
- "precision_at_50": 0.05059,
963
- "precision_at_100": 0.02756,
964
- "mrr_at_1": 0.6203125,
965
- "mrr_at_3": 0.7114583333333329,
966
- "mrr_at_5": 0.7251302083333327,
967
- "mrr_at_10": 0.7312574404761898,
968
- "mrr_at_20": 0.7330658025942703,
969
- "mrr_at_50": 0.7337640283895484,
970
- "mrr_at_100": 0.733871254289439,
971
- "naucs_at_1_max": 0.2039250412963779,
972
- "naucs_at_1_std": -0.07024884868861188,
973
- "naucs_at_1_diff1": 0.45606770903022215,
974
- "naucs_at_3_max": 0.058212985355640734,
975
- "naucs_at_3_std": -0.05959626421557797,
976
- "naucs_at_3_diff1": -0.029625222250297182,
977
- "naucs_at_5_max": 0.012523889018321476,
978
- "naucs_at_5_std": -0.05950139173785889,
979
- "naucs_at_5_diff1": -0.12093229795276991,
980
- "naucs_at_10_max": -0.07885246896142604,
981
- "naucs_at_10_std": -0.07584435766192119,
982
- "naucs_at_10_diff1": -0.19868179893511964,
983
- "naucs_at_20_max": -0.08104765093094657,
984
- "naucs_at_20_std": -0.020633911923048417,
985
- "naucs_at_20_diff1": -0.2329474014068329,
986
- "naucs_at_50_max": -0.10191575025539294,
987
- "naucs_at_50_std": 0.006497205696773289,
988
- "naucs_at_50_diff1": -0.2654796386435107,
989
- "naucs_at_100_max": -0.12912675673565918,
990
- "naucs_at_100_std": 0.002658413394124888,
991
- "naucs_at_100_diff1": -0.286740024806076
992
- },
993
- "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": {
994
- "ndcg_at_1": 0.56897,
995
- "ndcg_at_3": 0.55347,
996
- "ndcg_at_5": 0.54697,
997
- "ndcg_at_10": 0.54353,
998
- "ndcg_at_20": 0.5754,
999
- "ndcg_at_50": 0.64435,
1000
- "ndcg_at_100": 0.68217,
1001
- "map_at_1": 0.06995,
1002
- "map_at_3": 0.16788,
1003
- "map_at_5": 0.22262,
1004
- "map_at_10": 0.29539,
1005
- "map_at_20": 0.35806,
1006
- "map_at_50": 0.41596,
1007
- "map_at_100": 0.44377,
1008
- "recall_at_1": 0.06995,
1009
- "recall_at_3": 0.22001,
1010
- "recall_at_5": 0.31105,
1011
- "recall_at_10": 0.45156,
1012
- "recall_at_20": 0.60587,
1013
- "recall_at_50": 0.80629,
1014
- "recall_at_100": 0.91617,
1015
- "precision_at_1": 0.56897,
1016
- "precision_at_3": 0.52155,
1017
- "precision_at_5": 0.49483,
1018
- "precision_at_10": 0.4069,
1019
- "precision_at_20": 0.31207,
1020
- "precision_at_50": 0.19595,
1021
- "precision_at_100": 0.1275,
1022
- "mrr_at_1": 0.5689655172413793,
1023
- "mrr_at_3": 0.7040229885057472,
1024
- "mrr_at_5": 0.7173850574712642,
1025
- "mrr_at_10": 0.7212370005473451,
1026
- "mrr_at_20": 0.7221635921046082,
1027
- "mrr_at_50": 0.7223688466202075,
1028
- "mrr_at_100": 0.7223688466202075,
1029
- "naucs_at_1_max": 0.008287845664171043,
1030
- "naucs_at_1_std": 0.1276905868842376,
1031
- "naucs_at_1_diff1": 0.11685328065061831,
1032
- "naucs_at_3_max": 0.04231917732527594,
1033
- "naucs_at_3_std": 0.09097968258247123,
1034
- "naucs_at_3_diff1": 0.07039779015197889,
1035
- "naucs_at_5_max": 0.02450452954672108,
1036
- "naucs_at_5_std": 0.08382631941812721,
1037
- "naucs_at_5_diff1": 0.04903893807286711,
1038
- "naucs_at_10_max": 0.06258885863723426,
1039
- "naucs_at_10_std": 0.1489118063390922,
1040
- "naucs_at_10_diff1": -0.001772551183085018,
1041
- "naucs_at_20_max": 0.05127122134169797,
1042
- "naucs_at_20_std": 0.16207987797849627,
1043
- "naucs_at_20_diff1": -0.037084808056486104,
1044
- "naucs_at_50_max": 0.012079926402609917,
1045
- "naucs_at_50_std": 0.16924631618333094,
1046
- "naucs_at_50_diff1": -0.09727554327201936,
1047
- "naucs_at_100_max": -0.03658022842524374,
1048
- "naucs_at_100_std": 0.12849977057146159,
1049
- "naucs_at_100_diff1": -0.10217040873502983
1050
- },
1051
- "vidore/restaurant_esg_reports_beir": {
1052
- "ndcg_at_1": 0.71795,
1053
- "ndcg_at_3": 0.71188,
1054
- "ndcg_at_5": 0.73949,
1055
- "ndcg_at_10": 0.77205,
1056
- "ndcg_at_20": 0.78712,
1057
- "ndcg_at_50": 0.80866,
1058
- "ndcg_at_100": 0.81008,
1059
- "map_at_1": 0.50513,
1060
- "map_at_3": 0.62382,
1061
- "map_at_5": 0.6711,
1062
- "map_at_10": 0.70078,
1063
- "map_at_20": 0.7096,
1064
- "map_at_50": 0.71767,
1065
- "map_at_100": 0.71825,
1066
- "recall_at_1": 0.50513,
1067
- "recall_at_3": 0.68668,
1068
- "recall_at_5": 0.76906,
1069
- "recall_at_10": 0.85835,
1070
- "recall_at_20": 0.90725,
1071
- "recall_at_50": 0.97476,
1072
- "recall_at_100": 0.98071,
1073
- "precision_at_1": 0.73077,
1074
- "precision_at_3": 0.38462,
1075
- "precision_at_5": 0.28846,
1076
- "precision_at_10": 0.175,
1077
- "precision_at_20": 0.09712,
1078
- "precision_at_50": 0.045,
1079
- "precision_at_100": 0.02288,
1080
- "mrr_at_1": 0.75,
1081
- "mrr_at_3": 0.8076923076923077,
1082
- "mrr_at_5": 0.8125,
1083
- "mrr_at_10": 0.8226495726495725,
1084
- "mrr_at_20": 0.8226495726495725,
1085
- "mrr_at_50": 0.8233363858363858,
1086
- "mrr_at_100": 0.8233363858363858,
1087
- "naucs_at_1_max": 0.012222350607946745,
1088
- "naucs_at_1_std": -0.09466371539982948,
1089
- "naucs_at_1_diff1": 0.5022751223737485,
1090
- "naucs_at_3_max": -0.03148341001971903,
1091
- "naucs_at_3_std": -0.017366247697995007,
1092
- "naucs_at_3_diff1": -0.2806074619839212,
1093
- "naucs_at_5_max": -0.1368097107443235,
1094
- "naucs_at_5_std": -0.05686309477532154,
1095
- "naucs_at_5_diff1": -0.34542841772396893,
1096
- "naucs_at_10_max": -0.11047704445523453,
1097
- "naucs_at_10_std": 0.016760999608655385,
1098
- "naucs_at_10_diff1": -0.36219386821062893,
1099
- "naucs_at_20_max": -0.18118046303544919,
1100
- "naucs_at_20_std": 0.00660518472055942,
1101
- "naucs_at_20_diff1": -0.35322674833881423,
1102
- "naucs_at_50_max": -0.15496181424390146,
1103
- "naucs_at_50_std": 0.025351601144475155,
1104
- "naucs_at_50_diff1": -0.3271145822876798,
1105
- "naucs_at_100_max": -0.14539299033899847,
1106
- "naucs_at_100_std": 0.020343129158802924,
1107
- "naucs_at_100_diff1": -0.3237772572696072
1108
- }
1109
  }
1110
  }
 
1
  {
2
+ "vidore/arxivqa_test_subsampled":{
3
+ "ndcg_at_1":0.83,
4
+ "ndcg_at_3":0.87533,
5
+ "ndcg_at_5":0.88678,
6
+ "ndcg_at_10":0.89447,
7
+ "ndcg_at_20":0.89988,
8
+ "ndcg_at_50":0.90114,
9
+ "ndcg_at_100":0.90247,
10
+ "map_at_1":0.83,
11
+ "map_at_3":0.86467,
12
+ "map_at_5":0.87097,
13
+ "map_at_10":0.87409,
14
+ "map_at_20":0.87551,
15
+ "map_at_50":0.87574,
16
+ "map_at_100":0.87587,
17
+ "recall_at_1":0.83,
18
+ "recall_at_3":0.906,
19
+ "recall_at_5":0.934,
20
+ "recall_at_10":0.958,
21
+ "recall_at_20":0.98,
22
+ "recall_at_50":0.986,
23
+ "recall_at_100":0.994,
24
+ "precision_at_1":0.83,
25
+ "precision_at_3":0.302,
26
+ "precision_at_5":0.1868,
27
+ "precision_at_10":0.0958,
28
+ "precision_at_20":0.049,
29
+ "precision_at_50":0.01972,
30
+ "precision_at_100":0.00994,
31
+ "mrr_at_1":0.832,
32
+ "mrr_at_3":0.8663333333333333,
33
+ "mrr_at_5":0.8716333333333333,
34
+ "mrr_at_10":0.8752095238095239,
35
+ "mrr_at_20":0.8764481089945486,
36
+ "mrr_at_50":0.8769236116400513,
37
+ "mrr_at_100":0.8770087474728994,
38
+ "naucs_at_1_max":0.7799094444766933,
39
+ "naucs_at_1_std":-0.08580135833284933,
40
+ "naucs_at_1_diff1":0.9595692807801705,
41
+ "naucs_at_3_max":0.7955579394878525,
42
+ "naucs_at_3_std":-0.11155809841667234,
43
+ "naucs_at_3_diff1":0.9224725351133368,
44
+ "naucs_at_5_max":0.7705061822708856,
45
+ "naucs_at_5_std":-0.25026172084995735,
46
+ "naucs_at_5_diff1":0.9217666864725705,
47
+ "naucs_at_10_max":0.7739540260548654,
48
+ "naucs_at_10_std":-0.21324085189631833,
49
+ "naucs_at_10_diff1":0.9221910986616823,
50
+ "naucs_at_20_max":0.7708216619981358,
51
+ "naucs_at_20_std":-0.0996732026143794,
52
+ "naucs_at_20_diff1":0.8905228758169942,
53
+ "naucs_at_50_max":0.816593303988261,
54
+ "naucs_at_50_std":-0.19987995198079136,
55
+ "naucs_at_50_diff1":0.8832866479925231,
56
+ "naucs_at_100_max":0.9564270152505304,
57
+ "naucs_at_100_std":0.04310613134141329,
58
+ "naucs_at_100_diff1":0.9564270152505304
59
  },
60
+ "vidore/docvqa_test_subsampled":{
61
+ "ndcg_at_1":0.5388,
62
+ "ndcg_at_3":0.59542,
63
+ "ndcg_at_5":0.61269,
64
+ "ndcg_at_10":0.62919,
65
+ "ndcg_at_20":0.64259,
66
+ "ndcg_at_50":0.64929,
67
+ "ndcg_at_100":0.65683,
68
+ "map_at_1":0.5388,
69
+ "map_at_3":0.5813,
70
+ "map_at_5":0.59084,
71
+ "map_at_10":0.59764,
72
+ "map_at_20":0.6013,
73
+ "map_at_50":0.60241,
74
+ "map_at_100":0.60307,
75
+ "recall_at_1":0.5388,
76
+ "recall_at_3":0.63636,
77
+ "recall_at_5":0.67849,
78
+ "recall_at_10":0.72949,
79
+ "recall_at_20":0.78271,
80
+ "recall_at_50":0.81596,
81
+ "recall_at_100":0.86253,
82
+ "precision_at_1":0.5388,
83
+ "precision_at_3":0.21212,
84
+ "precision_at_5":0.1357,
85
+ "precision_at_10":0.07295,
86
+ "precision_at_20":0.03914,
87
+ "precision_at_50":0.01632,
88
+ "precision_at_100":0.00863,
89
+ "mrr_at_1":0.5432372505543237,
90
+ "mrr_at_3":0.5824094604582412,
91
+ "mrr_at_5":0.5903917220990392,
92
+ "mrr_at_10":0.5980097138633723,
93
+ "mrr_at_20":0.601820515422298,
94
+ "mrr_at_50":0.6029928998761527,
95
+ "mrr_at_100":0.6035667141572244,
96
+ "naucs_at_1_max":0.8430735352775856,
97
+ "naucs_at_1_std":0.5939128313043085,
98
+ "naucs_at_1_diff1":0.9018027917306838,
99
+ "naucs_at_3_max":0.8808230124613547,
100
+ "naucs_at_3_std":0.712681445860455,
101
+ "naucs_at_3_diff1":0.8646220134947054,
102
+ "naucs_at_5_max":0.8912698205476521,
103
+ "naucs_at_5_std":0.7525159249929689,
104
+ "naucs_at_5_diff1":0.8414611325660849,
105
+ "naucs_at_10_max":0.8852509111743405,
106
+ "naucs_at_10_std":0.817204422380503,
107
+ "naucs_at_10_diff1":0.8264628195305581,
108
+ "naucs_at_20_max":0.8512017211379178,
109
+ "naucs_at_20_std":0.8215729110215341,
110
+ "naucs_at_20_diff1":0.7992889048812711,
111
+ "naucs_at_50_max":0.8402551716801772,
112
+ "naucs_at_50_std":0.8733758748271155,
113
+ "naucs_at_50_diff1":0.7960833079573262,
114
+ "naucs_at_100_max":0.838262046357413,
115
+ "naucs_at_100_std":0.904184740422419,
116
+ "naucs_at_100_diff1":0.7996865824491111
117
+ },
118
+ "vidore/infovqa_test_subsampled":{
119
+ "ndcg_at_1":0.90081,
120
+ "ndcg_at_3":0.92934,
121
+ "ndcg_at_5":0.93352,
122
+ "ndcg_at_10":0.93669,
123
+ "ndcg_at_20":0.93871,
124
+ "ndcg_at_50":0.9403,
125
+ "ndcg_at_100":0.94099,
126
+ "map_at_1":0.90081,
127
+ "map_at_3":0.9224,
128
+ "map_at_5":0.92473,
129
+ "map_at_10":0.92598,
130
+ "map_at_20":0.92652,
131
+ "map_at_50":0.92677,
132
+ "map_at_100":0.92684,
133
+ "recall_at_1":0.90081,
134
+ "recall_at_3":0.94939,
135
+ "recall_at_5":0.95951,
136
+ "recall_at_10":0.96964,
137
+ "recall_at_20":0.97773,
138
+ "recall_at_50":0.98583,
139
+ "recall_at_100":0.98988,
140
+ "precision_at_1":0.90081,
141
+ "precision_at_3":0.31646,
142
+ "precision_at_5":0.1919,
143
+ "precision_at_10":0.09696,
144
+ "precision_at_20":0.04889,
145
+ "precision_at_50":0.01972,
146
+ "precision_at_100":0.0099,
147
+ "mrr_at_1":0.9008097165991903,
148
+ "mrr_at_3":0.9210526315789472,
149
+ "mrr_at_5":0.9248987854251012,
150
+ "mrr_at_10":0.9258040935672512,
151
+ "mrr_at_20":0.926239353314252,
152
+ "mrr_at_50":0.9265337558342984,
153
+ "mrr_at_100":0.9266429029872886,
154
+ "naucs_at_1_max":0.666687407456454,
155
+ "naucs_at_1_std":0.04348917328850024,
156
+ "naucs_at_1_diff1":0.9460185154989313,
157
+ "naucs_at_3_max":0.887692938235415,
158
+ "naucs_at_3_std":0.3724699632820944,
159
+ "naucs_at_3_diff1":0.9582075346992748,
160
+ "naucs_at_5_max":0.9121353932948266,
161
+ "naucs_at_5_std":0.46274982003896187,
162
+ "naucs_at_5_diff1":0.9608195637805694,
163
+ "naucs_at_10_max":0.9553640600031791,
164
+ "naucs_at_10_std":0.6527520622928011,
165
+ "naucs_at_10_diff1":0.9564661819784096,
166
+ "naucs_at_20_max":0.9643814216187027,
167
+ "naucs_at_20_std":0.8342696288850537,
168
+ "naucs_at_20_diff1":0.9643814216187027,
169
+ "naucs_at_50_max":1.0,
170
+ "naucs_at_50_std":0.960323608891707,
171
+ "naucs_at_50_diff1":0.9626852988386237,
172
+ "naucs_at_100_max":1.0,
173
+ "naucs_at_100_std":1.0,
174
+ "naucs_at_100_diff1":0.9477594183740937
175
+ },
176
+ "vidore/tabfquad_test_subsampled":{
177
+ "ndcg_at_1":0.925,
178
+ "ndcg_at_3":0.95561,
179
+ "ndcg_at_5":0.95991,
180
+ "ndcg_at_10":0.96464,
181
+ "ndcg_at_20":0.96464,
182
+ "ndcg_at_50":0.96464,
183
+ "ndcg_at_100":0.96464,
184
+ "map_at_1":0.925,
185
+ "map_at_3":0.94881,
186
+ "map_at_5":0.95113,
187
+ "map_at_10":0.95314,
188
+ "map_at_20":0.95314,
189
+ "map_at_50":0.95314,
190
+ "map_at_100":0.95314,
191
+ "recall_at_1":0.925,
192
+ "recall_at_3":0.975,
193
+ "recall_at_5":0.98571,
194
+ "recall_at_10":1.0,
195
+ "recall_at_20":1.0,
196
+ "recall_at_50":1.0,
197
+ "recall_at_100":1.0,
198
+ "precision_at_1":0.925,
199
+ "precision_at_3":0.325,
200
+ "precision_at_5":0.19714,
201
+ "precision_at_10":0.1,
202
+ "precision_at_20":0.05,
203
+ "precision_at_50":0.02,
204
+ "precision_at_100":0.01,
205
+ "mrr_at_1":0.9214285714285714,
206
+ "mrr_at_3":0.9464285714285714,
207
+ "mrr_at_5":0.9487499999999999,
208
+ "mrr_at_10":0.9507624716553289,
209
+ "mrr_at_20":0.9507624716553289,
210
+ "mrr_at_50":0.9507624716553289,
211
+ "mrr_at_100":0.9507624716553289,
212
+ "naucs_at_1_max":0.5344804588502072,
213
+ "naucs_at_1_std":0.26817393624116476,
214
+ "naucs_at_1_diff1":0.9383086567960518,
215
+ "naucs_at_3_max":0.9416433239962675,
216
+ "naucs_at_3_std":0.8989595838335351,
217
+ "naucs_at_3_diff1":0.98132586367881,
218
+ "naucs_at_5_max":0.967320261437913,
219
+ "naucs_at_5_std":0.8558590102707857,
220
+ "naucs_at_5_diff1":0.967320261437913,
221
+ "naucs_at_10_max":1.0,
222
+ "naucs_at_10_std":1.0,
223
+ "naucs_at_10_diff1":1.0,
224
+ "naucs_at_20_max":1.0,
225
+ "naucs_at_20_std":1.0,
226
+ "naucs_at_20_diff1":1.0,
227
+ "naucs_at_50_max":1.0,
228
+ "naucs_at_50_std":1.0,
229
+ "naucs_at_50_diff1":1.0,
230
+ "naucs_at_100_max":1.0,
231
+ "naucs_at_100_std":1.0,
232
+ "naucs_at_100_diff1":1.0
233
+ },
234
+ "vidore/tatdqa_test":{
235
+ "ndcg_at_1":0.70109,
236
+ "ndcg_at_3":0.79187,
237
+ "ndcg_at_5":0.81111,
238
+ "ndcg_at_10":0.82342,
239
+ "ndcg_at_20":0.8274,
240
+ "ndcg_at_50":0.83026,
241
+ "ndcg_at_100":0.83293,
242
+ "map_at_1":0.70109,
243
+ "map_at_3":0.77005,
244
+ "map_at_5":0.78071,
245
+ "map_at_10":0.78598,
246
+ "map_at_20":0.78706,
247
+ "map_at_50":0.78751,
248
+ "map_at_100":0.78775,
249
+ "recall_at_1":0.70109,
250
+ "recall_at_3":0.8548,
251
+ "recall_at_5":0.90158,
252
+ "recall_at_10":0.93864,
253
+ "recall_at_20":0.95443,
254
+ "recall_at_50":0.96902,
255
+ "recall_at_100":0.98542,
256
+ "precision_at_1":0.70109,
257
+ "precision_at_3":0.28493,
258
+ "precision_at_5":0.18032,
259
+ "precision_at_10":0.09386,
260
+ "precision_at_20":0.04772,
261
+ "precision_at_50":0.01938,
262
+ "precision_at_100":0.00985,
263
+ "mrr_at_1":0.7023086269744836,
264
+ "mrr_at_3":0.7714661806399354,
265
+ "mrr_at_5":0.782978938841637,
266
+ "mrr_at_10":0.7879385330478899,
267
+ "mrr_at_20":0.7889867733618092,
268
+ "mrr_at_50":0.7893770561483674,
269
+ "mrr_at_100":0.7896057919808999,
270
+ "naucs_at_1_max":0.25184895001768326,
271
+ "naucs_at_1_std":0.037017257538968965,
272
+ "naucs_at_1_diff1":0.8508843501202716,
273
+ "naucs_at_3_max":0.28836101906742934,
274
+ "naucs_at_3_std":0.0895587034833779,
275
+ "naucs_at_3_diff1":0.7677154118662606,
276
+ "naucs_at_5_max":0.33141107758129335,
277
+ "naucs_at_5_std":0.1402343443047445,
278
+ "naucs_at_5_diff1":0.726113129605695,
279
+ "naucs_at_10_max":0.3753484131980775,
280
+ "naucs_at_10_std":0.21181536222478634,
281
+ "naucs_at_10_diff1":0.7110698774619854,
282
+ "naucs_at_20_max":0.398127644970406,
283
+ "naucs_at_20_std":0.2900002928945751,
284
+ "naucs_at_20_diff1":0.6944425419244158,
285
+ "naucs_at_50_max":0.4094042090297014,
286
+ "naucs_at_50_std":0.3282254760208566,
287
+ "naucs_at_50_diff1":0.665997592036148,
288
+ "naucs_at_100_max":0.2601633836764814,
289
+ "naucs_at_100_std":0.2532897060083396,
290
+ "naucs_at_100_diff1":0.6808982214080673
291
+ },
292
+ "vidore/shiftproject_test":{
293
+ "ndcg_at_1":0.84,
294
+ "ndcg_at_3":0.91809,
295
+ "ndcg_at_5":0.91809,
296
+ "ndcg_at_10":0.92499,
297
+ "ndcg_at_20":0.92499,
298
+ "ndcg_at_50":0.92499,
299
+ "ndcg_at_100":0.92673,
300
+ "map_at_1":0.84,
301
+ "map_at_3":0.9,
302
+ "map_at_5":0.9,
303
+ "map_at_10":0.9031,
304
+ "map_at_20":0.9031,
305
+ "map_at_50":0.9031,
306
+ "map_at_100":0.90328,
307
+ "recall_at_1":0.84,
308
+ "recall_at_3":0.97,
309
+ "recall_at_5":0.97,
310
+ "recall_at_10":0.99,
311
+ "recall_at_20":0.99,
312
+ "recall_at_50":0.99,
313
+ "recall_at_100":1.0,
314
+ "precision_at_1":0.84,
315
+ "precision_at_3":0.32333,
316
+ "precision_at_5":0.194,
317
+ "precision_at_10":0.099,
318
+ "precision_at_20":0.0495,
319
+ "precision_at_50":0.0198,
320
+ "precision_at_100":0.01,
321
+ "mrr_at_1":0.84,
322
+ "mrr_at_3":0.8983333333333333,
323
+ "mrr_at_5":0.8983333333333333,
324
+ "mrr_at_10":0.9014285714285714,
325
+ "mrr_at_20":0.9014285714285714,
326
+ "mrr_at_50":0.9014285714285714,
327
+ "mrr_at_100":0.9016208791208791,
328
+ "naucs_at_1_max":0.1685076092292596,
329
+ "naucs_at_1_std":-0.2145925380461453,
330
+ "naucs_at_1_diff1":0.7751595483554242,
331
+ "naucs_at_3_max":0.6498599439775861,
332
+ "naucs_at_3_std":0.36834733893557625,
333
+ "naucs_at_3_diff1":0.664021164021167,
334
+ "naucs_at_5_max":0.6498599439775937,
335
+ "naucs_at_5_std":0.36834733893557176,
336
+ "naucs_at_5_diff1":0.6640211640211615,
337
+ "naucs_at_10_max":0.7222222222222276,
338
+ "naucs_at_10_std":0.5541549953314738,
339
+ "naucs_at_10_diff1":0.8692810457516413,
340
+ "naucs_at_20_max":0.7222222222222276,
341
+ "naucs_at_20_std":0.5541549953314738,
342
+ "naucs_at_20_diff1":0.8692810457516413,
343
+ "naucs_at_50_max":0.7222222222222041,
344
+ "naucs_at_50_std":0.554154995331464,
345
+ "naucs_at_50_diff1":0.8692810457516374,
346
+ "naucs_at_100_max":null,
347
+ "naucs_at_100_std":null,
348
+ "naucs_at_100_diff1":null
349
+ },
350
+ "vidore/syntheticDocQA_artificial_intelligence_test":{
351
+ "ndcg_at_1":0.98,
352
+ "ndcg_at_3":0.99262,
353
+ "ndcg_at_5":0.99262,
354
+ "ndcg_at_10":0.99262,
355
+ "ndcg_at_20":0.99262,
356
+ "ndcg_at_50":0.99262,
357
+ "ndcg_at_100":0.99262,
358
+ "map_at_1":0.98,
359
+ "map_at_3":0.99,
360
+ "map_at_5":0.99,
361
+ "map_at_10":0.99,
362
+ "map_at_20":0.99,
363
+ "map_at_50":0.99,
364
+ "map_at_100":0.99,
365
+ "recall_at_1":0.98,
366
+ "recall_at_3":1.0,
367
+ "recall_at_5":1.0,
368
+ "recall_at_10":1.0,
369
+ "recall_at_20":1.0,
370
+ "recall_at_50":1.0,
371
+ "recall_at_100":1.0,
372
+ "precision_at_1":0.98,
373
+ "precision_at_3":0.33333,
374
+ "precision_at_5":0.2,
375
+ "precision_at_10":0.1,
376
+ "precision_at_20":0.05,
377
+ "precision_at_50":0.02,
378
+ "precision_at_100":0.01,
379
+ "mrr_at_1":0.99,
380
+ "mrr_at_3":0.995,
381
+ "mrr_at_5":0.995,
382
+ "mrr_at_10":0.995,
383
+ "mrr_at_20":0.995,
384
+ "mrr_at_50":0.995,
385
+ "mrr_at_100":0.995,
386
+ "naucs_at_1_max":0.540149393090569,
387
+ "naucs_at_1_std":-0.6909430438842186,
388
+ "naucs_at_1_diff1":1.0,
389
+ "naucs_at_3_max":1.0,
390
+ "naucs_at_3_std":1.0,
391
+ "naucs_at_3_diff1":1.0,
392
+ "naucs_at_5_max":1.0,
393
+ "naucs_at_5_std":1.0,
394
+ "naucs_at_5_diff1":1.0,
395
+ "naucs_at_10_max":1.0,
396
+ "naucs_at_10_std":1.0,
397
+ "naucs_at_10_diff1":1.0,
398
+ "naucs_at_20_max":1.0,
399
+ "naucs_at_20_std":1.0,
400
+ "naucs_at_20_diff1":1.0,
401
+ "naucs_at_50_max":null,
402
+ "naucs_at_50_std":null,
403
+ "naucs_at_50_diff1":null,
404
+ "naucs_at_100_max":null,
405
+ "naucs_at_100_std":null,
406
+ "naucs_at_100_diff1":null
407
+ },
408
+ "vidore/syntheticDocQA_energy_test":{
409
+ "ndcg_at_1":0.95,
410
+ "ndcg_at_3":0.95631,
411
+ "ndcg_at_5":0.96492,
412
+ "ndcg_at_10":0.96781,
413
+ "ndcg_at_20":0.96781,
414
+ "ndcg_at_50":0.96976,
415
+ "ndcg_at_100":0.96976,
416
+ "map_at_1":0.95,
417
+ "map_at_3":0.955,
418
+ "map_at_5":0.96,
419
+ "map_at_10":0.961,
420
+ "map_at_20":0.961,
421
+ "map_at_50":0.96129,
422
+ "map_at_100":0.96129,
423
+ "recall_at_1":0.95,
424
+ "recall_at_3":0.96,
425
+ "recall_at_5":0.98,
426
+ "recall_at_10":0.99,
427
+ "recall_at_20":0.99,
428
+ "recall_at_50":1.0,
429
+ "recall_at_100":1.0,
430
+ "precision_at_1":0.95,
431
+ "precision_at_3":0.32,
432
+ "precision_at_5":0.196,
433
+ "precision_at_10":0.099,
434
+ "precision_at_20":0.0495,
435
+ "precision_at_50":0.02,
436
+ "precision_at_100":0.01,
437
+ "mrr_at_1":0.95,
438
+ "mrr_at_3":0.955,
439
+ "mrr_at_5":0.96,
440
+ "mrr_at_10":0.961,
441
+ "mrr_at_20":0.961,
442
+ "mrr_at_50":0.9613030303030302,
443
+ "mrr_at_100":0.9613030303030302,
444
+ "naucs_at_1_max":0.32362278244631343,
445
+ "naucs_at_1_std":-0.7713352007469629,
446
+ "naucs_at_1_diff1":1.0,
447
+ "naucs_at_3_max":0.8395191409897237,
448
+ "naucs_at_3_std":-0.6762371615312763,
449
+ "naucs_at_3_diff1":1.0,
450
+ "naucs_at_5_max":0.6790382819794609,
451
+ "naucs_at_5_std":-1.445845004668519,
452
+ "naucs_at_5_diff1":1.0,
453
+ "naucs_at_10_max":1.0,
454
+ "naucs_at_10_std":-1.1517273576097316,
455
+ "naucs_at_10_diff1":1.0,
456
+ "naucs_at_20_max":1.0,
457
+ "naucs_at_20_std":-1.1517273576097316,
458
+ "naucs_at_20_diff1":1.0,
459
+ "naucs_at_50_max":null,
460
+ "naucs_at_50_std":null,
461
+ "naucs_at_50_diff1":null,
462
+ "naucs_at_100_max":null,
463
+ "naucs_at_100_std":null,
464
+ "naucs_at_100_diff1":null
465
+ },
466
+ "vidore/syntheticDocQA_government_reports_test":{
467
+ "ndcg_at_1":0.91,
468
+ "ndcg_at_3":0.94655,
469
+ "ndcg_at_5":0.95085,
470
+ "ndcg_at_10":0.95743,
471
+ "ndcg_at_20":0.95743,
472
+ "ndcg_at_50":0.95743,
473
+ "ndcg_at_100":0.95743,
474
+ "map_at_1":0.91,
475
+ "map_at_3":0.93833,
476
+ "map_at_5":0.94083,
477
+ "map_at_10":0.94361,
478
+ "map_at_20":0.94361,
479
+ "map_at_50":0.94361,
480
+ "map_at_100":0.94361,
481
+ "recall_at_1":0.91,
482
+ "recall_at_3":0.97,
483
+ "recall_at_5":0.98,
484
+ "recall_at_10":1.0,
485
+ "recall_at_20":1.0,
486
+ "recall_at_50":1.0,
487
+ "recall_at_100":1.0,
488
+ "precision_at_1":0.91,
489
+ "precision_at_3":0.32333,
490
+ "precision_at_5":0.196,
491
+ "precision_at_10":0.1,
492
+ "precision_at_20":0.05,
493
+ "precision_at_50":0.02,
494
+ "precision_at_100":0.01,
495
+ "mrr_at_1":0.92,
496
+ "mrr_at_3":0.9433333333333332,
497
+ "mrr_at_5":0.9483333333333333,
498
+ "mrr_at_10":0.9494444444444444,
499
+ "mrr_at_20":0.9494444444444444,
500
+ "mrr_at_50":0.9494444444444444,
501
+ "mrr_at_100":0.9494444444444444,
502
+ "naucs_at_1_max":0.6774561676522453,
503
+ "naucs_at_1_std":0.14721444133208894,
504
+ "naucs_at_1_diff1":0.9564270152505436,
505
+ "naucs_at_3_max":0.8513849984438244,
506
+ "naucs_at_3_std":-0.24929971988795643,
507
+ "naucs_at_3_diff1":0.9564270152505466,
508
+ "naucs_at_5_max":1.0,
509
+ "naucs_at_5_std":-0.43534080298785716,
510
+ "naucs_at_5_diff1":0.9346405228758136,
511
+ "naucs_at_10_max":1.0,
512
+ "naucs_at_10_std":1.0,
513
+ "naucs_at_10_diff1":1.0,
514
+ "naucs_at_20_max":1.0,
515
+ "naucs_at_20_std":1.0,
516
+ "naucs_at_20_diff1":1.0,
517
+ "naucs_at_50_max":null,
518
+ "naucs_at_50_std":null,
519
+ "naucs_at_50_diff1":null,
520
+ "naucs_at_100_max":null,
521
+ "naucs_at_100_std":null,
522
+ "naucs_at_100_diff1":null
523
+ },
524
+ "vidore/syntheticDocQA_healthcare_industry_test":{
525
+ "ndcg_at_1":0.98,
526
+ "ndcg_at_3":0.99262,
527
+ "ndcg_at_5":0.99262,
528
+ "ndcg_at_10":0.99262,
529
+ "ndcg_at_20":0.99262,
530
+ "ndcg_at_50":0.99262,
531
+ "ndcg_at_100":0.99262,
532
+ "map_at_1":0.98,
533
+ "map_at_3":0.99,
534
+ "map_at_5":0.99,
535
+ "map_at_10":0.99,
536
+ "map_at_20":0.99,
537
+ "map_at_50":0.99,
538
+ "map_at_100":0.99,
539
+ "recall_at_1":0.98,
540
+ "recall_at_3":1.0,
541
+ "recall_at_5":1.0,
542
+ "recall_at_10":1.0,
543
+ "recall_at_20":1.0,
544
+ "recall_at_50":1.0,
545
+ "recall_at_100":1.0,
546
+ "precision_at_1":0.98,
547
+ "precision_at_3":0.33333,
548
+ "precision_at_5":0.2,
549
+ "precision_at_10":0.1,
550
+ "precision_at_20":0.05,
551
+ "precision_at_50":0.02,
552
+ "precision_at_100":0.01,
553
+ "mrr_at_1":0.98,
554
+ "mrr_at_3":0.99,
555
+ "mrr_at_5":0.99,
556
+ "mrr_at_10":0.99,
557
+ "mrr_at_20":0.99,
558
+ "mrr_at_50":0.99,
559
+ "mrr_at_100":0.99,
560
+ "naucs_at_1_max":0.7222222222222248,
561
+ "naucs_at_1_std":-0.5634920634920563,
562
+ "naucs_at_1_diff1":1.0,
563
+ "naucs_at_3_max":1.0,
564
+ "naucs_at_3_std":1.0,
565
+ "naucs_at_3_diff1":1.0,
566
+ "naucs_at_5_max":1.0,
567
+ "naucs_at_5_std":1.0,
568
+ "naucs_at_5_diff1":1.0,
569
+ "naucs_at_10_max":1.0,
570
+ "naucs_at_10_std":1.0,
571
+ "naucs_at_10_diff1":1.0,
572
+ "naucs_at_20_max":1.0,
573
+ "naucs_at_20_std":1.0,
574
+ "naucs_at_20_diff1":1.0,
575
+ "naucs_at_50_max":null,
576
+ "naucs_at_50_std":null,
577
+ "naucs_at_50_diff1":null,
578
+ "naucs_at_100_max":null,
579
+ "naucs_at_100_std":null,
580
+ "naucs_at_100_diff1":null
581
+ },
582
+ "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered":{
583
+ "ndcg_at_1":0.66875,
584
+ "ndcg_at_3":0.64833,
585
+ "ndcg_at_5":0.66136,
586
+ "ndcg_at_10":0.7002,
587
+ "ndcg_at_20":0.72343,
588
+ "ndcg_at_50":0.74221,
589
+ "ndcg_at_100":0.7504,
590
+ "map_at_1":0.41001,
591
+ "map_at_3":0.53585,
592
+ "map_at_5":0.57453,
593
+ "map_at_10":0.61456,
594
+ "map_at_20":0.63039,
595
+ "map_at_50":0.63734,
596
+ "map_at_100":0.63926,
597
+ "recall_at_1":0.41001,
598
+ "recall_at_3":0.59207,
599
+ "recall_at_5":0.66951,
600
+ "recall_at_10":0.7842,
601
+ "recall_at_20":0.84612,
602
+ "recall_at_50":0.90015,
603
+ "recall_at_100":0.92697,
604
+ "precision_at_1":0.66875,
605
+ "precision_at_3":0.39375,
606
+ "precision_at_5":0.29125,
607
+ "precision_at_10":0.18938,
608
+ "precision_at_20":0.11,
609
+ "precision_at_50":0.05087,
610
+ "precision_at_100":0.0275,
611
+ "mrr_at_1":0.65,
612
+ "mrr_at_3":0.7364583333333331,
613
+ "mrr_at_5":0.7480208333333331,
614
+ "mrr_at_10":0.7537127976190475,
615
+ "mrr_at_20":0.7549048402255638,
616
+ "mrr_at_50":0.755591370159045,
617
+ "mrr_at_100":0.755591370159045,
618
+ "naucs_at_1_max":0.3941383204398901,
619
+ "naucs_at_1_std":-0.03446929320007161,
620
+ "naucs_at_1_diff1":0.4503277010269023,
621
+ "naucs_at_3_max":0.029009734517562568,
622
+ "naucs_at_3_std":0.005402681474374929,
623
+ "naucs_at_3_diff1":-0.048706505991403734,
624
+ "naucs_at_5_max":-0.08207834225628742,
625
+ "naucs_at_5_std":-0.0032984001708723115,
626
+ "naucs_at_5_diff1":-0.17283336776461594,
627
+ "naucs_at_10_max":-0.17977285443640584,
628
+ "naucs_at_10_std":-0.04541684253923944,
629
+ "naucs_at_10_diff1":-0.23295672062273604,
630
+ "naucs_at_20_max":-0.170593905587224,
631
+ "naucs_at_20_std":0.015384406511413186,
632
+ "naucs_at_20_diff1":-0.2551053459454954,
633
+ "naucs_at_50_max":-0.22984435384481694,
634
+ "naucs_at_50_std":0.04443493398633732,
635
+ "naucs_at_50_diff1":-0.27937793023113594,
636
+ "naucs_at_100_max":-0.25957527630817795,
637
+ "naucs_at_100_std":0.057081217133354005,
638
+ "naucs_at_100_diff1":-0.3033742727110607
639
+ },
640
+ "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0":{
641
+ "ndcg_at_1":0.65517,
642
+ "ndcg_at_3":0.62816,
643
+ "ndcg_at_5":0.6159,
644
+ "ndcg_at_10":0.58912,
645
+ "ndcg_at_20":0.61706,
646
+ "ndcg_at_50":0.67765,
647
+ "ndcg_at_100":0.71471,
648
+ "map_at_1":0.08673,
649
+ "map_at_3":0.19894,
650
+ "map_at_5":0.26716,
651
+ "map_at_10":0.33292,
652
+ "map_at_20":0.40089,
653
+ "map_at_50":0.45667,
654
+ "map_at_100":0.4844,
655
+ "recall_at_1":0.08673,
656
+ "recall_at_3":0.24596,
657
+ "recall_at_5":0.34869,
658
+ "recall_at_10":0.4683,
659
+ "recall_at_20":0.62193,
660
+ "recall_at_50":0.80907,
661
+ "recall_at_100":0.92119,
662
+ "precision_at_1":0.65517,
663
+ "precision_at_3":0.58621,
664
+ "precision_at_5":0.55172,
665
+ "precision_at_10":0.42931,
666
+ "precision_at_20":0.32672,
667
+ "precision_at_50":0.19759,
668
+ "precision_at_100":0.1281,
669
+ "mrr_at_1":0.6896551724137931,
670
+ "mrr_at_3":0.7902298850574713,
671
+ "mrr_at_5":0.7971264367816091,
672
+ "mrr_at_10":0.8015051997810618,
673
+ "mrr_at_20":0.8015051997810618,
674
+ "mrr_at_50":0.8023262178434591,
675
+ "mrr_at_100":0.8023262178434591,
676
+ "naucs_at_1_max":-0.046307070115366736,
677
+ "naucs_at_1_std":0.15790688174002224,
678
+ "naucs_at_1_diff1":-0.14987305341556462,
679
+ "naucs_at_3_max":0.11671339502888234,
680
+ "naucs_at_3_std":0.11834362034275059,
681
+ "naucs_at_3_diff1":-0.17762652230301346,
682
+ "naucs_at_5_max":0.10679449683484428,
683
+ "naucs_at_5_std":0.09453878709557821,
684
+ "naucs_at_5_diff1":-0.1792193824994377,
685
+ "naucs_at_10_max":0.10651036963192122,
686
+ "naucs_at_10_std":0.207447480653671,
687
+ "naucs_at_10_diff1":-0.23752907816911517,
688
+ "naucs_at_20_max":0.05980777985340397,
689
+ "naucs_at_20_std":0.23365933479100473,
690
+ "naucs_at_20_diff1":-0.2025898492844735,
691
+ "naucs_at_50_max":-0.012623885278852142,
692
+ "naucs_at_50_std":0.2463485456131626,
693
+ "naucs_at_50_diff1":-0.18909262376408734,
694
+ "naucs_at_100_max":-0.08872391317584416,
695
+ "naucs_at_100_std":0.17948141514389135,
696
+ "naucs_at_100_diff1":-0.186930857132979
697
+ },
698
+ "vidore/synthetic_rse_restaurant_filtered_v1.0":{
699
+ "ndcg_at_1":0.52632,
700
+ "ndcg_at_3":0.52504,
701
+ "ndcg_at_5":0.57289,
702
+ "ndcg_at_10":0.62003,
703
+ "ndcg_at_20":0.65505,
704
+ "ndcg_at_50":0.68979,
705
+ "ndcg_at_100":0.70204,
706
+ "map_at_1":0.29474,
707
+ "map_at_3":0.40396,
708
+ "map_at_5":0.45972,
709
+ "map_at_10":0.50866,
710
+ "map_at_20":0.5276,
711
+ "map_at_50":0.54534,
712
+ "map_at_100":0.55161,
713
+ "recall_at_1":0.29474,
714
+ "recall_at_3":0.48191,
715
+ "recall_at_5":0.61543,
716
+ "recall_at_10":0.75002,
717
+ "recall_at_20":0.85537,
718
+ "recall_at_50":0.94841,
719
+ "recall_at_100":0.98246,
720
+ "precision_at_1":0.52632,
721
+ "precision_at_3":0.34503,
722
+ "precision_at_5":0.28421,
723
+ "precision_at_10":0.19649,
724
+ "precision_at_20":0.12281,
725
+ "precision_at_50":0.06912,
726
+ "precision_at_100":0.03877,
727
+ "mrr_at_1":0.543859649122807,
728
+ "mrr_at_3":0.6403508771929824,
729
+ "mrr_at_5":0.6526315789473685,
730
+ "mrr_at_10":0.6602548036758563,
731
+ "mrr_at_20":0.6651739829371409,
732
+ "mrr_at_50":0.6655248601301235,
733
+ "mrr_at_100":0.6655248601301235,
734
+ "naucs_at_1_max":0.07429395157545386,
735
+ "naucs_at_1_std":0.03015644215746205,
736
+ "naucs_at_1_diff1":0.029188519373247488,
737
+ "naucs_at_3_max":-0.14556834354246306,
738
+ "naucs_at_3_std":-0.03551874034674307,
739
+ "naucs_at_3_diff1":-0.03426575331185958,
740
+ "naucs_at_5_max":-0.132667608690031,
741
+ "naucs_at_5_std":0.046883073270420496,
742
+ "naucs_at_5_diff1":0.020789980948042533,
743
+ "naucs_at_10_max":-0.15350074281217774,
744
+ "naucs_at_10_std":0.021602996730365333,
745
+ "naucs_at_10_diff1":0.10610855445931551,
746
+ "naucs_at_20_max":-0.2870595843942612,
747
+ "naucs_at_20_std":-0.053200969005437526,
748
+ "naucs_at_20_diff1":0.009614869287472011,
749
+ "naucs_at_50_max":-0.3747840510110352,
750
+ "naucs_at_50_std":-0.1315278407645466,
751
+ "naucs_at_50_diff1":-0.08682413153737878,
752
+ "naucs_at_100_max":-0.3797677839881161,
753
+ "naucs_at_100_std":-0.11609328262961503,
754
+ "naucs_at_100_diff1":-0.12296867865157128
755
+ },
756
+ "vidore/synthetic_axa_filtered_v1.0":{
757
+ "ndcg_at_1":0.61111,
758
+ "ndcg_at_3":0.6462,
759
+ "ndcg_at_5":0.68292,
760
+ "ndcg_at_10":0.70017,
761
+ "ndcg_at_20":0.72201,
762
+ "ndcg_at_50":0.7613,
763
+ "ndcg_at_100":0.7658,
764
+ "map_at_1":0.29544,
765
+ "map_at_3":0.41914,
766
+ "map_at_5":0.49958,
767
+ "map_at_10":0.57784,
768
+ "map_at_20":0.61104,
769
+ "map_at_50":0.63144,
770
+ "map_at_100":0.63315,
771
+ "recall_at_1":0.29544,
772
+ "recall_at_3":0.51973,
773
+ "recall_at_5":0.66378,
774
+ "recall_at_10":0.78472,
775
+ "recall_at_20":0.85945,
776
+ "recall_at_50":0.97412,
777
+ "recall_at_100":0.98765,
778
+ "precision_at_1":0.61111,
779
+ "precision_at_3":0.46296,
780
+ "precision_at_5":0.42222,
781
+ "precision_at_10":0.31111,
782
+ "precision_at_20":0.19167,
783
+ "precision_at_50":0.09111,
784
+ "precision_at_100":0.04667,
785
+ "mrr_at_1":0.5555555555555556,
786
+ "mrr_at_3":0.6759259259259259,
787
+ "mrr_at_5":0.7037037037037037,
788
+ "mrr_at_10":0.7037037037037037,
789
+ "mrr_at_20":0.7037037037037037,
790
+ "mrr_at_50":0.7060185185185185,
791
+ "mrr_at_100":0.7060185185185185,
792
+ "naucs_at_1_max":0.06370640145678692,
793
+ "naucs_at_1_std":0.42153041887902504,
794
+ "naucs_at_1_diff1":0.35531111328419196,
795
+ "naucs_at_3_max":-0.49137089517804067,
796
+ "naucs_at_3_std":-0.41095639069482337,
797
+ "naucs_at_3_diff1":0.01707919337550571,
798
+ "naucs_at_5_max":-0.6120658234932744,
799
+ "naucs_at_5_std":-0.474389941591449,
800
+ "naucs_at_5_diff1":-0.165061956148533,
801
+ "naucs_at_10_max":-0.6883907404624792,
802
+ "naucs_at_10_std":-0.45513671453258775,
803
+ "naucs_at_10_diff1":-0.42316346500786917,
804
+ "naucs_at_20_max":-0.733957515102311,
805
+ "naucs_at_20_std":-0.4723325528538845,
806
+ "naucs_at_20_diff1":-0.4807537601912433,
807
+ "naucs_at_50_max":-0.7013881037720148,
808
+ "naucs_at_50_std":-0.45324759826825234,
809
+ "naucs_at_50_diff1":-0.4882986634795409,
810
+ "naucs_at_100_max":-0.6924248438890106,
811
+ "naucs_at_100_std":-0.4400487565870318,
812
+ "naucs_at_100_diff1":-0.4661845036852439
813
+ },
814
+ "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual":{
815
+ "ndcg_at_1":0.54825,
816
+ "ndcg_at_3":0.53008,
817
+ "ndcg_at_5":0.56705,
818
+ "ndcg_at_10":0.6257,
819
+ "ndcg_at_20":0.6625,
820
+ "ndcg_at_50":0.69353,
821
+ "ndcg_at_100":0.70465,
822
+ "map_at_1":0.29816,
823
+ "map_at_3":0.4016,
824
+ "map_at_5":0.4527,
825
+ "map_at_10":0.50738,
826
+ "map_at_20":0.52962,
827
+ "map_at_50":0.5466,
828
+ "map_at_100":0.55224,
829
+ "recall_at_1":0.29816,
830
+ "recall_at_3":0.48014,
831
+ "recall_at_5":0.59266,
832
+ "recall_at_10":0.76588,
833
+ "recall_at_20":0.87318,
834
+ "recall_at_50":0.95235,
835
+ "recall_at_100":0.98465,
836
+ "precision_at_1":0.54825,
837
+ "precision_at_3":0.34795,
838
+ "precision_at_5":0.28158,
839
+ "precision_at_10":0.19956,
840
+ "precision_at_20":0.12654,
841
+ "precision_at_50":0.06991,
842
+ "precision_at_100":0.03877,
843
+ "mrr_at_1":0.5394736842105263,
844
+ "mrr_at_3":0.6381578947368421,
845
+ "mrr_at_5":0.6528508771929824,
846
+ "mrr_at_10":0.666278543581175,
847
+ "mrr_at_20":0.6693822460076329,
848
+ "mrr_at_50":0.6696454039023697,
849
+ "mrr_at_100":0.6697038834345334,
850
+ "naucs_at_1_max":-0.10947777032499051,
851
+ "naucs_at_1_std":-0.08674361998375714,
852
+ "naucs_at_1_diff1":0.057249370353071344,
853
+ "naucs_at_3_max":-0.06687224875668692,
854
+ "naucs_at_3_std":0.0484364629589946,
855
+ "naucs_at_3_diff1":-0.05462426569089335,
856
+ "naucs_at_5_max":-0.08403937224497868,
857
+ "naucs_at_5_std":0.07293419772827552,
858
+ "naucs_at_5_diff1":-0.05958666027516894,
859
+ "naucs_at_10_max":-0.10838130867204888,
860
+ "naucs_at_10_std":0.05587982504285361,
861
+ "naucs_at_10_diff1":0.002192750308197299,
862
+ "naucs_at_20_max":-0.1628596976822229,
863
+ "naucs_at_20_std":0.06415536794747155,
864
+ "naucs_at_20_diff1":-0.0901131230050924,
865
+ "naucs_at_50_max":-0.2471444987505313,
866
+ "naucs_at_50_std":0.02178464235724526,
867
+ "naucs_at_50_diff1":-0.16484319783599452,
868
+ "naucs_at_100_max":-0.2476326734790779,
869
+ "naucs_at_100_std":0.04921366491713892,
870
+ "naucs_at_100_diff1":-0.1925485969942229
871
+ },
872
+ "vidore/synthetic_axa_filtered_v1.0_multilingual":{
873
+ "ndcg_at_1":0.56944,
874
+ "ndcg_at_3":0.60114,
875
+ "ndcg_at_5":0.61308,
876
+ "ndcg_at_10":0.63301,
877
+ "ndcg_at_20":0.65962,
878
+ "ndcg_at_50":0.70087,
879
+ "ndcg_at_100":0.71794,
880
+ "map_at_1":0.26807,
881
+ "map_at_3":0.38274,
882
+ "map_at_5":0.44543,
883
+ "map_at_10":0.51783,
884
+ "map_at_20":0.55091,
885
+ "map_at_50":0.57108,
886
+ "map_at_100":0.57464,
887
+ "recall_at_1":0.26807,
888
+ "recall_at_3":0.47345,
889
+ "recall_at_5":0.56537,
890
+ "recall_at_10":0.69703,
891
+ "recall_at_20":0.78343,
892
+ "recall_at_50":0.90152,
893
+ "recall_at_100":0.97889,
894
+ "precision_at_1":0.56944,
895
+ "precision_at_3":0.43056,
896
+ "precision_at_5":0.38056,
897
+ "precision_at_10":0.27917,
898
+ "precision_at_20":0.17569,
899
+ "precision_at_50":0.08583,
900
+ "precision_at_100":0.04569,
901
+ "mrr_at_1":0.5,
902
+ "mrr_at_3":0.6481481481481483,
903
+ "mrr_at_5":0.6550925925925927,
904
+ "mrr_at_10":0.6566358024691359,
905
+ "mrr_at_20":0.6577041785375121,
906
+ "mrr_at_50":0.659896412080552,
907
+ "mrr_at_100":0.6607831288544433,
908
+ "naucs_at_1_max":0.16468725740231643,
909
+ "naucs_at_1_std":0.37919017176717934,
910
+ "naucs_at_1_diff1":0.30214037198755495,
911
+ "naucs_at_3_max":-0.2419012234949494,
912
+ "naucs_at_3_std":-0.11880659562104584,
913
+ "naucs_at_3_diff1":-0.05983117053027332,
914
+ "naucs_at_5_max":-0.3435266789444122,
915
+ "naucs_at_5_std":-0.15122373891433766,
916
+ "naucs_at_5_diff1":-0.14301810021483025,
917
+ "naucs_at_10_max":-0.4049299119799346,
918
+ "naucs_at_10_std":-0.11551373909975436,
919
+ "naucs_at_10_diff1":-0.3087348082925975,
920
+ "naucs_at_20_max":-0.4557080881366001,
921
+ "naucs_at_20_std":-0.1316035057344777,
922
+ "naucs_at_20_diff1":-0.3767740930238455,
923
+ "naucs_at_50_max":-0.48221867587284106,
924
+ "naucs_at_50_std":-0.16520884486012927,
925
+ "naucs_at_50_diff1":-0.437539802211166,
926
+ "naucs_at_100_max":-0.5041035001503106,
927
+ "naucs_at_100_std":-0.19246332353391876,
928
+ "naucs_at_100_diff1":-0.450877397124322
929
+ },
930
+ "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual":{
931
+ "ndcg_at_1":0.62656,
932
+ "ndcg_at_3":0.62898,
933
+ "ndcg_at_5":0.64249,
934
+ "ndcg_at_10":0.67441,
935
+ "ndcg_at_20":0.69935,
936
+ "ndcg_at_50":0.72034,
937
+ "ndcg_at_100":0.7304,
938
+ "map_at_1":0.37553,
939
+ "map_at_3":0.51341,
940
+ "map_at_5":0.55134,
941
+ "map_at_10":0.58654,
942
+ "map_at_20":0.60275,
943
+ "map_at_50":0.61047,
944
+ "map_at_100":0.61272,
945
+ "recall_at_1":0.37553,
946
+ "recall_at_3":0.58317,
947
+ "recall_at_5":0.6651,
948
+ "recall_at_10":0.76267,
949
+ "recall_at_20":0.83047,
950
+ "recall_at_50":0.89402,
951
+ "recall_at_100":0.93005,
952
+ "precision_at_1":0.62656,
953
+ "precision_at_3":0.39115,
954
+ "precision_at_5":0.28812,
955
+ "precision_at_10":0.18453,
956
+ "precision_at_20":0.10859,
957
+ "precision_at_50":0.05059,
958
+ "precision_at_100":0.02756,
959
+ "mrr_at_1":0.6203125,
960
+ "mrr_at_3":0.7114583333333329,
961
+ "mrr_at_5":0.7251302083333327,
962
+ "mrr_at_10":0.7312574404761898,
963
+ "mrr_at_20":0.7330658025942703,
964
+ "mrr_at_50":0.7337640283895484,
965
+ "mrr_at_100":0.733871254289439,
966
+ "naucs_at_1_max":0.2039250412963779,
967
+ "naucs_at_1_std":-0.07024884868861188,
968
+ "naucs_at_1_diff1":0.45606770903022215,
969
+ "naucs_at_3_max":0.058212985355640734,
970
+ "naucs_at_3_std":-0.05959626421557797,
971
+ "naucs_at_3_diff1":-0.029625222250297182,
972
+ "naucs_at_5_max":0.012523889018321476,
973
+ "naucs_at_5_std":-0.05950139173785889,
974
+ "naucs_at_5_diff1":-0.12093229795276991,
975
+ "naucs_at_10_max":-0.07885246896142604,
976
+ "naucs_at_10_std":-0.07584435766192119,
977
+ "naucs_at_10_diff1":-0.19868179893511964,
978
+ "naucs_at_20_max":-0.08104765093094657,
979
+ "naucs_at_20_std":-0.020633911923048417,
980
+ "naucs_at_20_diff1":-0.2329474014068329,
981
+ "naucs_at_50_max":-0.10191575025539294,
982
+ "naucs_at_50_std":0.006497205696773289,
983
+ "naucs_at_50_diff1":-0.2654796386435107,
984
+ "naucs_at_100_max":-0.12912675673565918,
985
+ "naucs_at_100_std":0.002658413394124888,
986
+ "naucs_at_100_diff1":-0.286740024806076
987
+ },
988
+ "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual":{
989
+ "ndcg_at_1":0.56897,
990
+ "ndcg_at_3":0.55347,
991
+ "ndcg_at_5":0.54697,
992
+ "ndcg_at_10":0.54353,
993
+ "ndcg_at_20":0.5754,
994
+ "ndcg_at_50":0.64435,
995
+ "ndcg_at_100":0.68217,
996
+ "map_at_1":0.06995,
997
+ "map_at_3":0.16788,
998
+ "map_at_5":0.22262,
999
+ "map_at_10":0.29539,
1000
+ "map_at_20":0.35806,
1001
+ "map_at_50":0.41596,
1002
+ "map_at_100":0.44377,
1003
+ "recall_at_1":0.06995,
1004
+ "recall_at_3":0.22001,
1005
+ "recall_at_5":0.31105,
1006
+ "recall_at_10":0.45156,
1007
+ "recall_at_20":0.60587,
1008
+ "recall_at_50":0.80629,
1009
+ "recall_at_100":0.91617,
1010
+ "precision_at_1":0.56897,
1011
+ "precision_at_3":0.52155,
1012
+ "precision_at_5":0.49483,
1013
+ "precision_at_10":0.4069,
1014
+ "precision_at_20":0.31207,
1015
+ "precision_at_50":0.19595,
1016
+ "precision_at_100":0.1275,
1017
+ "mrr_at_1":0.5689655172413793,
1018
+ "mrr_at_3":0.7040229885057472,
1019
+ "mrr_at_5":0.7173850574712642,
1020
+ "mrr_at_10":0.7212370005473451,
1021
+ "mrr_at_20":0.7221635921046082,
1022
+ "mrr_at_50":0.7223688466202075,
1023
+ "mrr_at_100":0.7223688466202075,
1024
+ "naucs_at_1_max":0.008287845664171043,
1025
+ "naucs_at_1_std":0.1276905868842376,
1026
+ "naucs_at_1_diff1":0.11685328065061831,
1027
+ "naucs_at_3_max":0.04231917732527594,
1028
+ "naucs_at_3_std":0.09097968258247123,
1029
+ "naucs_at_3_diff1":0.07039779015197889,
1030
+ "naucs_at_5_max":0.02450452954672108,
1031
+ "naucs_at_5_std":0.08382631941812721,
1032
+ "naucs_at_5_diff1":0.04903893807286711,
1033
+ "naucs_at_10_max":0.06258885863723426,
1034
+ "naucs_at_10_std":0.1489118063390922,
1035
+ "naucs_at_10_diff1":-0.001772551183085018,
1036
+ "naucs_at_20_max":0.05127122134169797,
1037
+ "naucs_at_20_std":0.16207987797849627,
1038
+ "naucs_at_20_diff1":-0.037084808056486104,
1039
+ "naucs_at_50_max":0.012079926402609917,
1040
+ "naucs_at_50_std":0.16924631618333094,
1041
+ "naucs_at_50_diff1":-0.09727554327201936,
1042
+ "naucs_at_100_max":-0.03658022842524374,
1043
+ "naucs_at_100_std":0.12849977057146159,
1044
+ "naucs_at_100_diff1":-0.10217040873502983
1045
+ },
1046
+ "vidore/restaurant_esg_reports_beir":{
1047
+ "ndcg_at_1":0.71795,
1048
+ "ndcg_at_3":0.71188,
1049
+ "ndcg_at_5":0.73949,
1050
+ "ndcg_at_10":0.77205,
1051
+ "ndcg_at_20":0.78712,
1052
+ "ndcg_at_50":0.80866,
1053
+ "ndcg_at_100":0.81008,
1054
+ "map_at_1":0.50513,
1055
+ "map_at_3":0.62382,
1056
+ "map_at_5":0.6711,
1057
+ "map_at_10":0.70078,
1058
+ "map_at_20":0.7096,
1059
+ "map_at_50":0.71767,
1060
+ "map_at_100":0.71825,
1061
+ "recall_at_1":0.50513,
1062
+ "recall_at_3":0.68668,
1063
+ "recall_at_5":0.76906,
1064
+ "recall_at_10":0.85835,
1065
+ "recall_at_20":0.90725,
1066
+ "recall_at_50":0.97476,
1067
+ "recall_at_100":0.98071,
1068
+ "precision_at_1":0.73077,
1069
+ "precision_at_3":0.38462,
1070
+ "precision_at_5":0.28846,
1071
+ "precision_at_10":0.175,
1072
+ "precision_at_20":0.09712,
1073
+ "precision_at_50":0.045,
1074
+ "precision_at_100":0.02288,
1075
+ "mrr_at_1":0.75,
1076
+ "mrr_at_3":0.8076923076923077,
1077
+ "mrr_at_5":0.8125,
1078
+ "mrr_at_10":0.8226495726495725,
1079
+ "mrr_at_20":0.8226495726495725,
1080
+ "mrr_at_50":0.8233363858363858,
1081
+ "mrr_at_100":0.8233363858363858,
1082
+ "naucs_at_1_max":0.012222350607946745,
1083
+ "naucs_at_1_std":-0.09466371539982948,
1084
+ "naucs_at_1_diff1":0.5022751223737485,
1085
+ "naucs_at_3_max":-0.03148341001971903,
1086
+ "naucs_at_3_std":-0.017366247697995007,
1087
+ "naucs_at_3_diff1":-0.2806074619839212,
1088
+ "naucs_at_5_max":-0.1368097107443235,
1089
+ "naucs_at_5_std":-0.05686309477532154,
1090
+ "naucs_at_5_diff1":-0.34542841772396893,
1091
+ "naucs_at_10_max":-0.11047704445523453,
1092
+ "naucs_at_10_std":0.016760999608655385,
1093
+ "naucs_at_10_diff1":-0.36219386821062893,
1094
+ "naucs_at_20_max":-0.18118046303544919,
1095
+ "naucs_at_20_std":0.00660518472055942,
1096
+ "naucs_at_20_diff1":-0.35322674833881423,
1097
+ "naucs_at_50_max":-0.15496181424390146,
1098
+ "naucs_at_50_std":0.025351601144475155,
1099
+ "naucs_at_50_diff1":-0.3271145822876798,
1100
+ "naucs_at_100_max":-0.14539299033899847,
1101
+ "naucs_at_100_std":0.020343129158802924,
1102
+ "naucs_at_100_diff1":-0.3237772572696072
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1103
  }
1104
  }