metadata
base_model:
- Alibaba-NLP/gte-Qwen2-7B-instruct
language:
- en
- zh
license: apache-2.0
model-index:
- name: infly/inf-retriever-v1
results:
- dataset:
config: default
name: MTEB CmedqaRetrieval (default)
revision: cd540c506dae1cf9e9a59c3e06f42030d54e7301
split: dev
type: C-MTEB/CmedqaRetrieval
metrics:
- type: ndcg_at_1
value: 38.185
- type: ndcg_at_3
value: 38.438
- type: ndcg_at_5
value: 40.445
- type: ndcg_at_10
value: 43.308
- type: ndcg_at_20
value: 46.177
- type: ndcg_at_100
value: 50.644999999999996
- type: ndcg_at_1000
value: 52.819
- type: recall_at_1
value: 25.14
- type: recall_at_3
value: 38.253
- type: recall_at_5
value: 44.507999999999996
- type: recall_at_10
value: 53.025
- type: recall_at_20
value: 62.89
- type: recall_at_100
value: 83.487
- type: recall_at_1000
value: 98.059
- type: main_score
value: 43.308
task:
type: Retrieval
- dataset:
config: default
name: MTEB CovidRetrieval (default)
revision: 1271c7809071a13532e05f25fb53511ffce77117
split: dev
type: C-MTEB/CovidRetrieval
metrics:
- type: ndcg_at_1
value: 77.97699999999999
- type: ndcg_at_3
value: 85.24199999999999
- type: ndcg_at_5
value: 86.901
- type: ndcg_at_10
value: 87.77000000000001
- type: ndcg_at_20
value: 88.295
- type: ndcg_at_100
value: 88.479
- type: ndcg_at_1000
value: 88.527
- type: recall_at_1
value: 77.819
- type: recall_at_3
value: 89.96300000000001
- type: recall_at_5
value: 93.941
- type: recall_at_10
value: 96.575
- type: recall_at_20
value: 98.63
- type: recall_at_100
value: 99.579
- type: recall_at_1000
value: 100
- type: main_score
value: 87.77000000000001
task:
type: Retrieval
- dataset:
config: default
name: MTEB DuRetrieval (default)
revision: a1a333e290fe30b10f3f56498e3a0d911a693ced
split: dev
type: C-MTEB/DuRetrieval
metrics:
- type: ndcg_at_1
value: 91.45
- type: ndcg_at_3
value: 89.249
- type: ndcg_at_5
value: 88.506
- type: ndcg_at_10
value: 90.66
- type: ndcg_at_20
value: 91.886
- type: ndcg_at_100
value: 92.78699999999999
- type: ndcg_at_1000
value: 92.944
- type: recall_at_1
value: 27.332
- type: recall_at_3
value: 61.07599999999999
- type: recall_at_5
value: 78.49199999999999
- type: recall_at_10
value: 92.002
- type: recall_at_20
value: 96.116
- type: recall_at_100
value: 99.009
- type: recall_at_1000
value: 99.844
- type: main_score
value: 90.66
task:
type: Retrieval
- dataset:
config: default
name: MTEB EcomRetrieval (default)
revision: 687de13dc7294d6fd9be10c6945f9e8fec8166b9
split: dev
type: C-MTEB/EcomRetrieval
metrics:
- type: ndcg_at_1
value: 55.900000000000006
- type: ndcg_at_3
value: 66.019
- type: ndcg_at_5
value: 68.47999999999999
- type: ndcg_at_10
value: 70.678
- type: ndcg_at_20
value: 72.024
- type: ndcg_at_100
value: 72.933
- type: ndcg_at_1000
value: 73.20400000000001
- type: recall_at_1
value: 55.900000000000006
- type: recall_at_3
value: 73.1
- type: recall_at_5
value: 79.10000000000001
- type: recall_at_10
value: 85.9
- type: recall_at_20
value: 91.2
- type: recall_at_100
value: 96.1
- type: recall_at_1000
value: 98.3
- type: main_score
value: 70.678
task:
type: Retrieval
- dataset:
config: default
name: MTEB MMarcoRetrieval (default)
revision: 539bbde593d947e2a124ba72651aafc09eb33fc2
split: dev
type: C-MTEB/MMarcoRetrieval
metrics:
- type: ndcg_at_1
value: 75.74499999999999
- type: ndcg_at_3
value: 82.188
- type: ndcg_at_5
value: 83.869
- type: ndcg_at_10
value: 85.119
- type: ndcg_at_20
value: 85.624
- type: ndcg_at_100
value: 86.051
- type: ndcg_at_1000
value: 86.177
- type: recall_at_1
value: 73.33
- type: recall_at_3
value: 86.823
- type: recall_at_5
value: 90.814
- type: recall_at_10
value: 94.509
- type: recall_at_20
value: 96.422
- type: recall_at_100
value: 98.6
- type: recall_at_1000
value: 99.599
- type: main_score
value: 85.119
task:
type: Retrieval
- dataset:
config: default
name: MTEB MedicalRetrieval (default)
revision: 2039188fb5800a9803ba5048df7b76e6fb151fc6
split: dev
type: C-MTEB/MedicalRetrieval
metrics:
- type: ndcg_at_1
value: 55.00000000000001
- type: ndcg_at_3
value: 61.334
- type: ndcg_at_5
value: 62.590999999999994
- type: ndcg_at_10
value: 63.913
- type: ndcg_at_20
value: 64.748
- type: ndcg_at_100
value: 66.675
- type: ndcg_at_1000
value: 67.894
- type: recall_at_1
value: 55.00000000000001
- type: recall_at_3
value: 65.60000000000001
- type: recall_at_5
value: 68.60000000000001
- type: recall_at_10
value: 72.7
- type: recall_at_20
value: 76
- type: recall_at_100
value: 86.6
- type: recall_at_1000
value: 96.3
- type: main_score
value: 63.913
task:
type: Retrieval
- dataset:
config: default
name: MTEB T2Retrieval (default)
revision: 8731a845f1bf500a4f111cf1070785c793d10e64
split: dev
type: C-MTEB/T2Retrieval
metrics:
- type: ndcg_at_1
value: 91.526
- type: ndcg_at_3
value: 88.35499999999999
- type: ndcg_at_5
value: 87.408
- type: ndcg_at_10
value: 87.641
- type: ndcg_at_20
value: 89.265
- type: ndcg_at_100
value: 90.693
- type: ndcg_at_1000
value: 91.105
- type: recall_at_1
value: 28.359
- type: recall_at_3
value: 58.101
- type: recall_at_5
value: 72.99
- type: recall_at_10
value: 86.921
- type: recall_at_20
value: 92.497
- type: recall_at_100
value: 96.978
- type: recall_at_1000
value: 99.075
- type: main_score
value: 87.641
task:
type: Retrieval
- dataset:
config: default
name: MTEB VideoRetrieval (default)
revision: 58c2597a5943a2ba48f4668c3b90d796283c5639
split: dev
type: C-MTEB/VideoRetrieval
metrics:
- type: ndcg_at_1
value: 66
- type: ndcg_at_3
value: 75.495
- type: ndcg_at_5
value: 77.027
- type: ndcg_at_10
value: 78.606
- type: ndcg_at_20
value: 79.54599999999999
- type: ndcg_at_100
value: 80.326
- type: ndcg_at_1000
value: 80.516
- type: recall_at_1
value: 66
- type: recall_at_3
value: 81.89999999999999
- type: recall_at_5
value: 85.6
- type: recall_at_10
value: 90.4
- type: recall_at_20
value: 94.1
- type: recall_at_100
value: 98.2
- type: recall_at_1000
value: 99.7
- type: main_score
value: 78.606
task:
type: Retrieval
- dataset:
config: default
name: MTEB AILACasedocs (default)
revision: 4106e6bcc72e0698d714ea8b101355e3e238431a
split: test
type: mteb/AILA_casedocs
metrics:
- type: ndcg_at_1
value: 40
- type: ndcg_at_3
value: 37.37
- type: ndcg_at_5
value: 37.913999999999994
- type: ndcg_at_10
value: 41.162
- type: ndcg_at_20
value: 45.72
- type: ndcg_at_100
value: 54.126
- type: ndcg_at_1000
value: 55.907
- type: recall_at_1
value: 15.406
- type: recall_at_3
value: 26.56
- type: recall_at_5
value: 33.084
- type: recall_at_10
value: 45.972
- type: recall_at_20
value: 60.775
- type: recall_at_100
value: 91.105
- type: recall_at_1000
value: 100
- type: main_score
value: 41.162
task:
type: Retrieval
- dataset:
config: default
name: MTEB AILAStatutes (default)
revision: ebfcd844eadd3d667efa3c57fc5c8c87f5c2867e
split: test
type: mteb/AILA_statutes
metrics:
- type: ndcg_at_1
value: 36
- type: ndcg_at_3
value: 32.427
- type: ndcg_at_5
value: 31.512
- type: ndcg_at_10
value: 37.727
- type: ndcg_at_20
value: 43.808
- type: ndcg_at_100
value: 56.445
- type: ndcg_at_1000
value: 56.445
- type: recall_at_1
value: 8.1
- type: recall_at_3
value: 20.599999999999998
- type: recall_at_5
value: 30.733
- type: recall_at_10
value: 42.733
- type: recall_at_20
value: 57.733000000000004
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: main_score
value: 37.727
task:
type: Retrieval
- dataset:
config: default
name: MTEB AlloprofRetrieval (default)
revision: fcf295ea64c750f41fadbaa37b9b861558e1bfbd
split: test
type: lyon-nlp/alloprof
metrics:
- type: ndcg_at_1
value: 45.509
- type: ndcg_at_3
value: 57.912
- type: ndcg_at_5
value: 60.885
- type: ndcg_at_10
value: 63.611
- type: ndcg_at_20
value: 64.976
- type: ndcg_at_100
value: 66.507
- type: ndcg_at_1000
value: 66.998
- type: recall_at_1
value: 45.509
- type: recall_at_3
value: 66.537
- type: recall_at_5
value: 73.748
- type: recall_at_10
value: 82.16799999999999
- type: recall_at_20
value: 87.522
- type: recall_at_100
value: 95.72500000000001
- type: recall_at_1000
value: 99.655
- type: main_score
value: 63.611
task:
type: Retrieval
- dataset:
config: default
name: MTEB AppsRetrieval (default)
revision: f22508f96b7a36c2415181ed8bb76f76e04ae2d5
split: test
type: CoIR-Retrieval/apps
metrics:
- type: ndcg_at_1
value: 35.405
- type: ndcg_at_3
value: 42.945
- type: ndcg_at_5
value: 44.984
- type: ndcg_at_10
value: 47.369
- type: ndcg_at_20
value: 49.095
- type: ndcg_at_100
value: 51.821
- type: ndcg_at_1000
value: 53.581
- type: recall_at_1
value: 35.405
- type: recall_at_3
value: 48.287
- type: recall_at_5
value: 53.227000000000004
- type: recall_at_10
value: 60.611000000000004
- type: recall_at_20
value: 67.437
- type: recall_at_100
value: 82.231
- type: recall_at_1000
value: 96.38799999999999
- type: main_score
value: 47.369
task:
type: Retrieval
- dataset:
config: default
name: MTEB ArguAna (default)
revision: c22ab2a51041ffd869aaddef7af8d8215647e41a
split: test
type: mteb/arguana
metrics:
- type: ndcg_at_1
value: 69.132
- type: ndcg_at_3
value: 81.661
- type: ndcg_at_5
value: 83.773
- type: ndcg_at_10
value: 84.855
- type: ndcg_at_20
value: 85.073
- type: ndcg_at_100
value: 85.134
- type: ndcg_at_1000
value: 85.134
- type: recall_at_1
value: 69.132
- type: recall_at_3
value: 90.185
- type: recall_at_5
value: 95.235
- type: recall_at_10
value: 98.506
- type: recall_at_20
value: 99.36
- type: recall_at_100
value: 99.644
- type: recall_at_1000
value: 99.644
- type: main_score
value: 84.855
task:
type: Retrieval
- dataset:
config: default
name: MTEB ArguAna-PL (default)
revision: 63fc86750af76253e8c760fc9e534bbf24d260a2
split: test
type: clarin-knext/arguana-pl
metrics:
- type: ndcg_at_1
value: 46.657
- type: ndcg_at_3
value: 63.388999999999996
- type: ndcg_at_5
value: 67.931
- type: ndcg_at_10
value: 70.745
- type: ndcg_at_20
value: 71.60300000000001
- type: ndcg_at_100
value: 71.941
- type: ndcg_at_1000
value: 71.961
- type: recall_at_1
value: 46.657
- type: recall_at_3
value: 75.036
- type: recall_at_5
value: 85.989
- type: recall_at_10
value: 94.523
- type: recall_at_20
value: 97.795
- type: recall_at_100
value: 99.502
- type: recall_at_1000
value: 99.644
- type: main_score
value: 70.745
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackAndroidRetrieval (default)
revision: f46a197baaae43b4f621051089b82a364682dfeb
split: test
type: mteb/cqadupstack-android
metrics:
- type: ndcg_at_1
value: 45.494
- type: ndcg_at_3
value: 51.53
- type: ndcg_at_5
value: 54.062
- type: ndcg_at_10
value: 56.599
- type: ndcg_at_20
value: 58.663
- type: ndcg_at_100
value: 61.36200000000001
- type: ndcg_at_1000
value: 62.824000000000005
- type: recall_at_1
value: 37.078
- type: recall_at_3
value: 53.529
- type: recall_at_5
value: 60.772999999999996
- type: recall_at_10
value: 68.65299999999999
- type: recall_at_20
value: 75.92999999999999
- type: recall_at_100
value: 88.127
- type: recall_at_1000
value: 97.059
- type: main_score
value: 56.599
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackEnglishRetrieval (default)
revision: ad9991cb51e31e31e430383c75ffb2885547b5f0
split: test
type: mteb/cqadupstack-english
metrics:
- type: ndcg_at_1
value: 47.134
- type: ndcg_at_3
value: 52.186
- type: ndcg_at_5
value: 53.94
- type: ndcg_at_10
value: 55.96
- type: ndcg_at_20
value: 57.521
- type: ndcg_at_100
value: 59.865
- type: ndcg_at_1000
value: 61.611000000000004
- type: recall_at_1
value: 37.405
- type: recall_at_3
value: 53.869
- type: recall_at_5
value: 59.18600000000001
- type: recall_at_10
value: 65.786
- type: recall_at_20
value: 71.56099999999999
- type: recall_at_100
value: 82.062
- type: recall_at_1000
value: 92.863
- type: main_score
value: 55.96
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackGamingRetrieval (default)
revision: 4885aa143210c98657558c04aaf3dc47cfb54340
split: test
type: mteb/cqadupstack-gaming
metrics:
- type: ndcg_at_1
value: 52.22599999999999
- type: ndcg_at_3
value: 59.797999999999995
- type: ndcg_at_5
value: 62.260000000000005
- type: ndcg_at_10
value: 64.85300000000001
- type: ndcg_at_20
value: 66.398
- type: ndcg_at_100
value: 68.298
- type: ndcg_at_1000
value: 69.003
- type: recall_at_1
value: 45.789
- type: recall_at_3
value: 64.9
- type: recall_at_5
value: 70.902
- type: recall_at_10
value: 78.388
- type: recall_at_20
value: 84.086
- type: recall_at_100
value: 93.006
- type: recall_at_1000
value: 97.928
- type: main_score
value: 64.85300000000001
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackGisRetrieval (default)
revision: 5003b3064772da1887988e05400cf3806fe491f2
split: test
type: mteb/cqadupstack-gis
metrics:
- type: ndcg_at_1
value: 32.09
- type: ndcg_at_3
value: 38.339
- type: ndcg_at_5
value: 41.427
- type: ndcg_at_10
value: 43.606
- type: ndcg_at_20
value: 45.784000000000006
- type: ndcg_at_100
value: 48.908
- type: ndcg_at_1000
value: 50.585
- type: recall_at_1
value: 29.146
- type: recall_at_3
value: 43.168
- type: recall_at_5
value: 50.717
- type: recall_at_10
value: 57.120000000000005
- type: recall_at_20
value: 65.254
- type: recall_at_100
value: 81.04599999999999
- type: recall_at_1000
value: 93.487
- type: main_score
value: 43.606
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackMathematicaRetrieval (default)
revision: 90fceea13679c63fe563ded68f3b6f06e50061de
split: test
type: mteb/cqadupstack-mathematica
metrics:
- type: ndcg_at_1
value: 24.876
- type: ndcg_at_3
value: 29.663
- type: ndcg_at_5
value: 32.193
- type: ndcg_at_10
value: 34.694
- type: ndcg_at_20
value: 37.075
- type: ndcg_at_100
value: 40.615
- type: ndcg_at_1000
value: 43.317
- type: recall_at_1
value: 20.395
- type: recall_at_3
value: 32.521
- type: recall_at_5
value: 38.887
- type: recall_at_10
value: 46.388
- type: recall_at_20
value: 54.885
- type: recall_at_100
value: 71.597
- type: recall_at_1000
value: 90.75
- type: main_score
value: 34.694
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackPhysicsRetrieval (default)
revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4
split: test
type: mteb/cqadupstack-physics
metrics:
- type: ndcg_at_1
value: 42.733
- type: ndcg_at_3
value: 47.236
- type: ndcg_at_5
value: 49.327
- type: ndcg_at_10
value: 52.346000000000004
- type: ndcg_at_20
value: 54.446000000000005
- type: ndcg_at_100
value: 57.736
- type: ndcg_at_1000
value: 59.245000000000005
- type: recall_at_1
value: 34.414
- type: recall_at_3
value: 50.233000000000004
- type: recall_at_5
value: 55.967
- type: recall_at_10
value: 65.173
- type: recall_at_20
value: 72.27799999999999
- type: recall_at_100
value: 87.163
- type: recall_at_1000
value: 96.64
- type: main_score
value: 52.346000000000004
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackProgrammersRetrieval (default)
revision: 6184bc1440d2dbc7612be22b50686b8826d22b32
split: test
type: mteb/cqadupstack-programmers
metrics:
- type: ndcg_at_1
value: 37.329
- type: ndcg_at_3
value: 41.319
- type: ndcg_at_5
value: 43.444
- type: ndcg_at_10
value: 46.643
- type: ndcg_at_20
value: 49.257
- type: ndcg_at_100
value: 52.524
- type: ndcg_at_1000
value: 54.478
- type: recall_at_1
value: 30.278
- type: recall_at_3
value: 43.464999999999996
- type: recall_at_5
value: 49.419999999999995
- type: recall_at_10
value: 58.650999999999996
- type: recall_at_20
value: 67.90899999999999
- type: recall_at_100
value: 83.276
- type: recall_at_1000
value: 96.114
- type: main_score
value: 46.643
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackRetrieval (default)
revision: CQADupstackRetrieval_is_a_combined_dataset
split: test
type: CQADupstackRetrieval_is_a_combined_dataset
metrics:
- type: main_score
value: 46.644083333333334
- type: ndcg_at_10
value: 46.644083333333334
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackStatsRetrieval (default)
revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a
split: test
type: mteb/cqadupstack-stats
metrics:
- type: ndcg_at_1
value: 30.368000000000002
- type: ndcg_at_3
value: 35.004000000000005
- type: ndcg_at_5
value: 37.125
- type: ndcg_at_10
value: 39.831
- type: ndcg_at_20
value: 42.099
- type: ndcg_at_100
value: 45.032
- type: ndcg_at_1000
value: 47.016999999999996
- type: recall_at_1
value: 27.151999999999997
- type: recall_at_3
value: 38.2
- type: recall_at_5
value: 43.349
- type: recall_at_10
value: 51.50599999999999
- type: recall_at_20
value: 60.035000000000004
- type: recall_at_100
value: 74.869
- type: recall_at_1000
value: 89.159
- type: main_score
value: 39.831
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackTexRetrieval (default)
revision: 46989137a86843e03a6195de44b09deda022eec7
split: test
type: mteb/cqadupstack-tex
metrics:
- type: ndcg_at_1
value: 26.222
- type: ndcg_at_3
value: 30.085
- type: ndcg_at_5
value: 31.977
- type: ndcg_at_10
value: 34.107
- type: ndcg_at_20
value: 35.939
- type: ndcg_at_100
value: 39.054
- type: ndcg_at_1000
value: 41.899
- type: recall_at_1
value: 21.552
- type: recall_at_3
value: 32.66
- type: recall_at_5
value: 37.785000000000004
- type: recall_at_10
value: 44.143
- type: recall_at_20
value: 50.968999999999994
- type: recall_at_100
value: 66.392
- type: recall_at_1000
value: 86.601
- type: main_score
value: 34.107
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackUnixRetrieval (default)
revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53
split: test
type: mteb/cqadupstack-unix
metrics:
- type: ndcg_at_1
value: 36.287000000000006
- type: ndcg_at_3
value: 41.15
- type: ndcg_at_5
value: 43.283
- type: ndcg_at_10
value: 45.698
- type: ndcg_at_20
value: 47.754000000000005
- type: ndcg_at_100
value: 50.800999999999995
- type: ndcg_at_1000
value: 53.024
- type: recall_at_1
value: 30.791
- type: recall_at_3
value: 44.802
- type: recall_at_5
value: 50.434999999999995
- type: recall_at_10
value: 57.424
- type: recall_at_20
value: 64.702
- type: recall_at_100
value: 79.216
- type: recall_at_1000
value: 94.602
- type: main_score
value: 45.698
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackWebmastersRetrieval (default)
revision: 160c094312a0e1facb97e55eeddb698c0abe3571
split: test
type: mteb/cqadupstack-webmasters
metrics:
- type: ndcg_at_1
value: 37.352000000000004
- type: ndcg_at_3
value: 43.029
- type: ndcg_at_5
value: 44.811
- type: ndcg_at_10
value: 47.493
- type: ndcg_at_20
value: 49.76
- type: ndcg_at_100
value: 52.925
- type: ndcg_at_1000
value: 55.117000000000004
- type: recall_at_1
value: 31.719
- type: recall_at_3
value: 45.466
- type: recall_at_5
value: 50.087
- type: recall_at_10
value: 57.86
- type: recall_at_20
value: 66.27
- type: recall_at_100
value: 81.437
- type: recall_at_1000
value: 95.162
- type: main_score
value: 47.493
task:
type: Retrieval
- dataset:
config: default
name: MTEB CQADupstackWordpressRetrieval (default)
revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4
split: test
type: mteb/cqadupstack-wordpress
metrics:
- type: ndcg_at_1
value: 29.020000000000003
- type: ndcg_at_3
value: 33.715
- type: ndcg_at_5
value: 35.266
- type: ndcg_at_10
value: 37.899
- type: ndcg_at_20
value: 39.812999999999995
- type: ndcg_at_100
value: 42.998999999999995
- type: ndcg_at_1000
value: 45.257
- type: recall_at_1
value: 26.784000000000002
- type: recall_at_3
value: 37.049
- type: recall_at_5
value: 40.638000000000005
- type: recall_at_10
value: 48.204
- type: recall_at_20
value: 55.496
- type: recall_at_100
value: 71.749
- type: recall_at_1000
value: 88.22
- type: main_score
value: 37.899
task:
type: Retrieval
- dataset:
config: default
name: MTEB CodeFeedbackMT (default)
revision: b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f
split: test
type: CoIR-Retrieval/codefeedback-mt
metrics:
- type: ndcg_at_1
value: 67.214
- type: ndcg_at_3
value: 74.774
- type: ndcg_at_5
value: 76.297
- type: ndcg_at_10
value: 77.644
- type: ndcg_at_20
value: 78.41
- type: ndcg_at_100
value: 79.374
- type: ndcg_at_1000
value: 79.77
- type: recall_at_1
value: 67.214
- type: recall_at_3
value: 79.95
- type: recall_at_5
value: 83.65599999999999
- type: recall_at_10
value: 87.776
- type: recall_at_20
value: 90.781
- type: recall_at_100
value: 95.993
- type: recall_at_1000
value: 99.104
- type: main_score
value: 77.644
task:
type: Retrieval
- dataset:
config: default
name: MTEB CodeFeedbackST (default)
revision: d213819e87aab9010628da8b73ab4eb337c89340
split: test
type: CoIR-Retrieval/codefeedback-st
metrics:
- type: ndcg_at_1
value: 74.05000000000001
- type: ndcg_at_3
value: 84.59
- type: ndcg_at_5
value: 85.949
- type: ndcg_at_10
value: 86.627
- type: ndcg_at_20
value: 86.907
- type: ndcg_at_100
value: 87.149
- type: ndcg_at_1000
value: 87.21799999999999
- type: recall_at_1
value: 74.05000000000001
- type: recall_at_3
value: 91.685
- type: recall_at_5
value: 94.959
- type: recall_at_10
value: 97.017
- type: recall_at_20
value: 98.10900000000001
- type: recall_at_100
value: 99.396
- type: recall_at_1000
value: 99.92699999999999
- type: main_score
value: 86.627
task:
type: Retrieval
- dataset:
config: python
name: MTEB CodeSearchNetCCRetrieval (python)
revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8
split: test
type: CoIR-Retrieval/CodeSearchNet-ccr
metrics:
- type: ndcg_at_1
value: 69.875
- type: ndcg_at_3
value: 79.45100000000001
- type: ndcg_at_5
value: 80.95400000000001
- type: ndcg_at_10
value: 82.025
- type: ndcg_at_20
value: 82.526
- type: ndcg_at_100
value: 83.07
- type: ndcg_at_1000
value: 83.28999999999999
- type: recall_at_1
value: 69.875
- type: recall_at_3
value: 85.957
- type: recall_at_5
value: 89.59
- type: recall_at_10
value: 92.874
- type: recall_at_20
value: 94.838
- type: recall_at_100
value: 97.748
- type: recall_at_1000
value: 99.47
- type: main_score
value: 82.025
task:
type: Retrieval
- dataset:
config: javascript
name: MTEB CodeSearchNetCCRetrieval (javascript)
revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8
split: test
type: CoIR-Retrieval/CodeSearchNet-ccr
metrics:
- type: ndcg_at_1
value: 66.18
- type: ndcg_at_3
value: 76.294
- type: ndcg_at_5
value: 77.849
- type: ndcg_at_10
value: 78.95400000000001
- type: ndcg_at_20
value: 79.71000000000001
- type: ndcg_at_100
value: 80.402
- type: ndcg_at_1000
value: 80.694
- type: recall_at_1
value: 66.18
- type: recall_at_3
value: 83.10499999999999
- type: recall_at_5
value: 86.873
- type: recall_at_10
value: 90.277
- type: recall_at_20
value: 93.22399999999999
- type: recall_at_100
value: 96.87
- type: recall_at_1000
value: 99.21
- type: main_score
value: 78.95400000000001
task:
type: Retrieval
- dataset:
config: go
name: MTEB CodeSearchNetCCRetrieval (go)
revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8
split: test
type: CoIR-Retrieval/CodeSearchNet-ccr
metrics:
- type: ndcg_at_1
value: 57.24
- type: ndcg_at_3
value: 67.84700000000001
- type: ndcg_at_5
value: 70.126
- type: ndcg_at_10
value: 71.839
- type: ndcg_at_20
value: 72.89
- type: ndcg_at_100
value: 73.904
- type: ndcg_at_1000
value: 74.343
- type: recall_at_1
value: 57.24
- type: recall_at_3
value: 75.179
- type: recall_at_5
value: 80.67
- type: recall_at_10
value: 85.939
- type: recall_at_20
value: 90.076
- type: recall_at_100
value: 95.48100000000001
- type: recall_at_1000
value: 98.929
- type: main_score
value: 71.839
task:
type: Retrieval
- dataset:
config: ruby
name: MTEB CodeSearchNetCCRetrieval (ruby)
revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8
split: test
type: CoIR-Retrieval/CodeSearchNet-ccr
metrics:
- type: ndcg_at_1
value: 64.235
- type: ndcg_at_3
value: 73.451
- type: ndcg_at_5
value: 75.233
- type: ndcg_at_10
value: 76.53
- type: ndcg_at_20
value: 77.35
- type: ndcg_at_100
value: 78.13799999999999
- type: ndcg_at_1000
value: 78.57
- type: recall_at_1
value: 64.235
- type: recall_at_3
value: 79.699
- type: recall_at_5
value: 83.981
- type: recall_at_10
value: 88.02499999999999
- type: recall_at_20
value: 91.277
- type: recall_at_100
value: 95.638
- type: recall_at_1000
value: 99.048
- type: main_score
value: 76.53
task:
type: Retrieval
- dataset:
config: java
name: MTEB CodeSearchNetCCRetrieval (java)
revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8
split: test
type: CoIR-Retrieval/CodeSearchNet-ccr
metrics:
- type: ndcg_at_1
value: 65.468
- type: ndcg_at_3
value: 75.064
- type: ndcg_at_5
value: 76.786
- type: ndcg_at_10
value: 77.929
- type: ndcg_at_20
value: 78.596
- type: ndcg_at_100
value: 79.28699999999999
- type: ndcg_at_1000
value: 79.625
- type: recall_at_1
value: 65.468
- type: recall_at_3
value: 81.56099999999999
- type: recall_at_5
value: 85.714
- type: recall_at_10
value: 89.229
- type: recall_at_20
value: 91.83
- type: recall_at_100
value: 95.509
- type: recall_at_1000
value: 98.17399999999999
- type: main_score
value: 77.929
task:
type: Retrieval
- dataset:
config: php
name: MTEB CodeSearchNetCCRetrieval (php)
revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8
split: test
type: CoIR-Retrieval/CodeSearchNet-ccr
metrics:
- type: ndcg_at_1
value: 52.71900000000001
- type: ndcg_at_3
value: 63.025
- type: ndcg_at_5
value: 65.17399999999999
- type: ndcg_at_10
value: 66.982
- type: ndcg_at_20
value: 68.113
- type: ndcg_at_100
value: 69.443
- type: ndcg_at_1000
value: 70.111
- type: recall_at_1
value: 52.71900000000001
- type: recall_at_3
value: 70.158
- type: recall_at_5
value: 75.35300000000001
- type: recall_at_10
value: 80.919
- type: recall_at_20
value: 85.36500000000001
- type: recall_at_100
value: 92.486
- type: recall_at_1000
value: 97.788
- type: main_score
value: 66.982
task:
type: Retrieval
- dataset:
config: python
name: MTEB CodeSearchNetRetrieval (python)
revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759
split: test
type: code-search-net/code_search_net
metrics:
- type: ndcg_at_1
value: 86.9
- type: ndcg_at_3
value: 92.012
- type: ndcg_at_5
value: 93.002
- type: ndcg_at_10
value: 93.304
- type: ndcg_at_20
value: 93.432
- type: ndcg_at_100
value: 93.50500000000001
- type: ndcg_at_1000
value: 93.54
- type: recall_at_1
value: 86.9
- type: recall_at_3
value: 95.5
- type: recall_at_5
value: 97.89999999999999
- type: recall_at_10
value: 98.8
- type: recall_at_20
value: 99.3
- type: recall_at_100
value: 99.7
- type: recall_at_1000
value: 100
- type: main_score
value: 93.304
task:
type: Retrieval
- dataset:
config: javascript
name: MTEB CodeSearchNetRetrieval (javascript)
revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759
split: test
type: code-search-net/code_search_net
metrics:
- type: ndcg_at_1
value: 73.9
- type: ndcg_at_3
value: 80.297
- type: ndcg_at_5
value: 81.162
- type: ndcg_at_10
value: 82.075
- type: ndcg_at_20
value: 82.432
- type: ndcg_at_100
value: 82.948
- type: ndcg_at_1000
value: 83.722
- type: recall_at_1
value: 73.9
- type: recall_at_3
value: 84.6
- type: recall_at_5
value: 86.7
- type: recall_at_10
value: 89.5
- type: recall_at_20
value: 90.9
- type: recall_at_100
value: 93.7
- type: recall_at_1000
value: 100
- type: main_score
value: 82.075
task:
type: Retrieval
- dataset:
config: go
name: MTEB CodeSearchNetRetrieval (go)
revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759
split: test
type: code-search-net/code_search_net
metrics:
- type: ndcg_at_1
value: 86.9
- type: ndcg_at_3
value: 92.961
- type: ndcg_at_5
value: 93.632
- type: ndcg_at_10
value: 93.865
- type: ndcg_at_20
value: 93.917
- type: ndcg_at_100
value: 93.994
- type: ndcg_at_1000
value: 94.02199999999999
- type: recall_at_1
value: 86.9
- type: recall_at_3
value: 96.89999999999999
- type: recall_at_5
value: 98.5
- type: recall_at_10
value: 99.2
- type: recall_at_20
value: 99.4
- type: recall_at_100
value: 99.8
- type: recall_at_1000
value: 100
- type: main_score
value: 93.865
task:
type: Retrieval
- dataset:
config: ruby
name: MTEB CodeSearchNetRetrieval (ruby)
revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759
split: test
type: code-search-net/code_search_net
metrics:
- type: ndcg_at_1
value: 79.10000000000001
- type: ndcg_at_3
value: 85.626
- type: ndcg_at_5
value: 86.629
- type: ndcg_at_10
value: 87.16000000000001
- type: ndcg_at_20
value: 87.414
- type: ndcg_at_100
value: 87.7
- type: ndcg_at_1000
value: 88.115
- type: recall_at_1
value: 79.10000000000001
- type: recall_at_3
value: 89.9
- type: recall_at_5
value: 92.30000000000001
- type: recall_at_10
value: 93.89999999999999
- type: recall_at_20
value: 94.89999999999999
- type: recall_at_100
value: 96.39999999999999
- type: recall_at_1000
value: 100
- type: main_score
value: 87.16000000000001
task:
type: Retrieval
- dataset:
config: java
name: MTEB CodeSearchNetRetrieval (java)
revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759
split: test
type: code-search-net/code_search_net
metrics:
- type: ndcg_at_1
value: 82
- type: ndcg_at_3
value: 89.205
- type: ndcg_at_5
value: 89.86699999999999
- type: ndcg_at_10
value: 90.269
- type: ndcg_at_20
value: 90.32
- type: ndcg_at_100
value: 90.36999999999999
- type: ndcg_at_1000
value: 90.691
- type: recall_at_1
value: 82
- type: recall_at_3
value: 94
- type: recall_at_5
value: 95.6
- type: recall_at_10
value: 96.8
- type: recall_at_20
value: 97
- type: recall_at_100
value: 97.3
- type: recall_at_1000
value: 100
- type: main_score
value: 90.269
task:
type: Retrieval
- dataset:
config: php
name: MTEB CodeSearchNetRetrieval (php)
revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759
split: test
type: code-search-net/code_search_net
metrics:
- type: ndcg_at_1
value: 76.1
- type: ndcg_at_3
value: 83.97
- type: ndcg_at_5
value: 85.128
- type: ndcg_at_10
value: 85.922
- type: ndcg_at_20
value: 86.279
- type: ndcg_at_100
value: 86.53
- type: ndcg_at_1000
value: 86.846
- type: recall_at_1
value: 76.1
- type: recall_at_3
value: 89.3
- type: recall_at_5
value: 92.10000000000001
- type: recall_at_10
value: 94.5
- type: recall_at_20
value: 95.89999999999999
- type: recall_at_100
value: 97.3
- type: recall_at_1000
value: 100
- type: main_score
value: 85.922
task:
type: Retrieval
- dataset:
config: default
name: MTEB CodeTransOceanContest (default)
revision: 20da4eb20a4b17300c0986ee148c90867a7f2a4d
split: test
type: CoIR-Retrieval/codetrans-contest
metrics:
- type: ndcg_at_1
value: 82.353
- type: ndcg_at_3
value: 86.792
- type: ndcg_at_5
value: 88.116
- type: ndcg_at_10
value: 89.164
- type: ndcg_at_20
value: 89.627
- type: ndcg_at_100
value: 89.816
- type: ndcg_at_1000
value: 89.929
- type: recall_at_1
value: 82.353
- type: recall_at_3
value: 90.045
- type: recall_at_5
value: 93.21300000000001
- type: recall_at_10
value: 96.38
- type: recall_at_20
value: 98.19
- type: recall_at_100
value: 99.095
- type: recall_at_1000
value: 100
- type: main_score
value: 89.164
task:
type: Retrieval
- dataset:
config: default
name: MTEB CodeTransOceanDL (default)
revision: 281562cb8a1265ab5c0824bfa6ddcd9b0a15618f
split: test
type: CoIR-Retrieval/codetrans-dl
metrics:
- type: ndcg_at_1
value: 9.443999999999999
- type: ndcg_at_3
value: 13.141
- type: ndcg_at_5
value: 20.149
- type: ndcg_at_10
value: 35.181000000000004
- type: ndcg_at_20
value: 39.898
- type: ndcg_at_100
value: 40.337
- type: ndcg_at_1000
value: 40.337
- type: recall_at_1
value: 9.443999999999999
- type: recall_at_3
value: 16.111
- type: recall_at_5
value: 32.778
- type: recall_at_10
value: 80.55600000000001
- type: recall_at_20
value: 97.77799999999999
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: main_score
value: 35.181000000000004
task:
type: Retrieval
- dataset:
config: default
name: MTEB CosQA (default)
revision: bc5efb7e9d437246ce393ed19d772e08e4a79535
split: test
type: CoIR-Retrieval/cosqa
metrics:
- type: ndcg_at_1
value: 14.2
- type: ndcg_at_3
value: 23.647000000000002
- type: ndcg_at_5
value: 28.655
- type: ndcg_at_10
value: 34.175
- type: ndcg_at_20
value: 37.04
- type: ndcg_at_100
value: 41.074
- type: ndcg_at_1000
value: 41.917
- type: recall_at_1
value: 14.2
- type: recall_at_3
value: 31
- type: recall_at_5
value: 43.4
- type: recall_at_10
value: 60.4
- type: recall_at_20
value: 71.8
- type: recall_at_100
value: 93
- type: recall_at_1000
value: 99.2
- type: main_score
value: 34.175
task:
type: Retrieval
- dataset:
config: default
name: MTEB DBPedia (default)
revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659
split: test
type: mteb/dbpedia
metrics:
- type: ndcg_at_1
value: 61.5
- type: ndcg_at_3
value: 53.476
- type: ndcg_at_5
value: 51.601
- type: ndcg_at_10
value: 50.391
- type: ndcg_at_20
value: 49.342000000000006
- type: ndcg_at_100
value: 55.37800000000001
- type: ndcg_at_1000
value: 62.470000000000006
- type: recall_at_1
value: 9.757
- type: recall_at_3
value: 17.203
- type: recall_at_5
value: 21.878
- type: recall_at_10
value: 30.425
- type: recall_at_20
value: 39.137
- type: recall_at_100
value: 62.885000000000005
- type: recall_at_1000
value: 85.795
- type: main_score
value: 50.391
task:
type: Retrieval
- dataset:
config: default
name: MTEB FiQA-PL (default)
revision: 2e535829717f8bf9dc829b7f911cc5bbd4e6608e
split: test
type: clarin-knext/fiqa-pl
metrics:
- type: ndcg_at_1
value: 46.296
- type: ndcg_at_3
value: 43.682
- type: ndcg_at_5
value: 44.818999999999996
- type: ndcg_at_10
value: 47.137
- type: ndcg_at_20
value: 49.957
- type: ndcg_at_100
value: 53.998999999999995
- type: ndcg_at_1000
value: 56.547000000000004
- type: recall_at_1
value: 23.116999999999997
- type: recall_at_3
value: 39.967000000000006
- type: recall_at_5
value: 46.745
- type: recall_at_10
value: 54.202
- type: recall_at_20
value: 62.61600000000001
- type: recall_at_100
value: 79.322
- type: recall_at_1000
value: 94.114
- type: main_score
value: 47.137
task:
type: Retrieval
- dataset:
config: default
name: MTEB FiQA2018 (default)
revision: 27a168819829fe9bcd655c2df245fb19452e8e06
split: test
type: mteb/fiqa
metrics:
- type: ndcg_at_1
value: 63.117000000000004
- type: ndcg_at_3
value: 58.538999999999994
- type: ndcg_at_5
value: 59.147000000000006
- type: ndcg_at_10
value: 62.35000000000001
- type: ndcg_at_20
value: 65.36800000000001
- type: ndcg_at_100
value: 68.801
- type: ndcg_at_1000
value: 70.06599999999999
- type: recall_at_1
value: 33.377
- type: recall_at_3
value: 52.817
- type: recall_at_5
value: 59.03699999999999
- type: recall_at_10
value: 69.116
- type: recall_at_20
value: 78.30799999999999
- type: recall_at_100
value: 91.715
- type: recall_at_1000
value: 98.783
- type: main_score
value: 62.35000000000001
task:
type: Retrieval
- dataset:
config: default
name: MTEB GerDaLIRSmall (default)
revision: 48327de6ee192e9610f3069789719788957c7abd
split: test
type: mteb/GerDaLIRSmall
metrics:
- type: ndcg_at_1
value: 30.047
- type: ndcg_at_3
value: 36.635
- type: ndcg_at_5
value: 39.237
- type: ndcg_at_10
value: 41.752
- type: ndcg_at_20
value: 43.467
- type: ndcg_at_100
value: 45.793
- type: ndcg_at_1000
value: 47.404
- type: recall_at_1
value: 27.272999999999996
- type: recall_at_3
value: 41.534
- type: recall_at_5
value: 47.678
- type: recall_at_10
value: 55.131
- type: recall_at_20
value: 61.592
- type: recall_at_100
value: 73.604
- type: recall_at_1000
value: 86.146
- type: main_score
value: 41.752
task:
type: Retrieval
- dataset:
config: default
name: MTEB HotpotQA (default)
revision: ab518f4d6fcca38d87c25209f94beba119d02014
split: test
type: mteb/hotpotqa
metrics:
- type: ndcg_at_1
value: 88.062
- type: ndcg_at_3
value: 77.443
- type: ndcg_at_5
value: 80.05600000000001
- type: ndcg_at_10
value: 81.979
- type: ndcg_at_20
value: 83.033
- type: ndcg_at_100
value: 84.232
- type: ndcg_at_1000
value: 84.827
- type: recall_at_1
value: 44.031
- type: recall_at_3
value: 75.71900000000001
- type: recall_at_5
value: 80.851
- type: recall_at_10
value: 85.652
- type: recall_at_20
value: 89.021
- type: recall_at_100
value: 94.267
- type: recall_at_1000
value: 98.136
- type: main_score
value: 81.979
task:
type: Retrieval
- dataset:
config: default
name: MTEB LEMBNarrativeQARetrieval (default)
revision: 6e346642246bfb4928c560ee08640dc84d074e8c
split: test
type: dwzhu/LongEmbed
metrics:
- type: ndcg_at_1
value: 49.009
- type: ndcg_at_3
value: 56.69
- type: ndcg_at_5
value: 58.572
- type: ndcg_at_10
value: 60.702
- type: ndcg_at_20
value: 62.160000000000004
- type: ndcg_at_100
value: 64.461
- type: ndcg_at_1000
value: 65.604
- type: recall_at_1
value: 49.009
- type: recall_at_3
value: 62.073
- type: recall_at_5
value: 66.648
- type: recall_at_10
value: 73.222
- type: recall_at_20
value: 78.974
- type: recall_at_100
value: 91.444
- type: recall_at_1000
value: 100
- type: main_score
value: 60.702
task:
type: Retrieval
- dataset:
config: default
name: MTEB LEMBNeedleRetrieval (default)
revision: 6e346642246bfb4928c560ee08640dc84d074e8c
split: test_256
type: dwzhu/LongEmbed
metrics:
- type: ndcg_at_1
value: 8
- type: ndcg_at_3
value: 12.786
- type: ndcg_at_5
value: 15.282000000000002
- type: ndcg_at_10
value: 20.096
- type: ndcg_at_20
value: 22.631
- type: ndcg_at_100
value: 32.174
- type: ndcg_at_1000
value: 32.174
- type: recall_at_1
value: 8
- type: recall_at_3
value: 16
- type: recall_at_5
value: 22
- type: recall_at_10
value: 36
- type: recall_at_20
value: 46
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: main_score
value: 8
task:
type: Retrieval
- dataset:
config: default
name: MTEB LEMBPasskeyRetrieval (default)
revision: 6e346642246bfb4928c560ee08640dc84d074e8c
split: test_256
type: dwzhu/LongEmbed
metrics:
- type: ndcg_at_1
value: 10
- type: ndcg_at_3
value: 12
- type: ndcg_at_5
value: 12
- type: ndcg_at_10
value: 12.631
- type: ndcg_at_20
value: 14.982000000000001
- type: ndcg_at_100
value: 28.534
- type: ndcg_at_1000
value: 28.534
- type: recall_at_1
value: 10
- type: recall_at_3
value: 14.000000000000002
- type: recall_at_5
value: 14.000000000000002
- type: recall_at_10
value: 16
- type: recall_at_20
value: 26
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: main_score
value: 10
task:
type: Retrieval
- dataset:
config: default
name: MTEB LEMBQMSumRetrieval (default)
revision: 6e346642246bfb4928c560ee08640dc84d074e8c
split: test
type: dwzhu/LongEmbed
metrics:
- type: ndcg_at_1
value: 37.263000000000005
- type: ndcg_at_3
value: 48.207
- type: ndcg_at_5
value: 51.464
- type: ndcg_at_10
value: 55.071999999999996
- type: ndcg_at_20
value: 57.364000000000004
- type: ndcg_at_100
value: 60.236999999999995
- type: ndcg_at_1000
value: 60.352
- type: recall_at_1
value: 37.263000000000005
- type: recall_at_3
value: 55.92700000000001
- type: recall_at_5
value: 63.851
- type: recall_at_10
value: 74.91799999999999
- type: recall_at_20
value: 83.955
- type: recall_at_100
value: 99.214
- type: recall_at_1000
value: 100
- type: main_score
value: 55.071999999999996
task:
type: Retrieval
- dataset:
config: default
name: MTEB LEMBSummScreenFDRetrieval (default)
revision: 6e346642246bfb4928c560ee08640dc84d074e8c
split: validation
type: dwzhu/LongEmbed
metrics:
- type: ndcg_at_1
value: 94.345
- type: ndcg_at_3
value: 96.66900000000001
- type: ndcg_at_5
value: 97.297
- type: ndcg_at_10
value: 97.387
- type: ndcg_at_20
value: 97.387
- type: ndcg_at_100
value: 97.387
- type: ndcg_at_1000
value: 97.387
- type: recall_at_1
value: 94.345
- type: recall_at_3
value: 98.214
- type: recall_at_5
value: 99.702
- type: recall_at_10
value: 100
- type: recall_at_20
value: 100
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: main_score
value: 97.387
task:
type: Retrieval
- dataset:
config: default
name: MTEB LEMBWikimQARetrieval (default)
revision: 6e346642246bfb4928c560ee08640dc84d074e8c
split: test
type: dwzhu/LongEmbed
metrics:
- type: ndcg_at_1
value: 80
- type: ndcg_at_3
value: 84.024
- type: ndcg_at_5
value: 84.985
- type: ndcg_at_10
value: 85.751
- type: ndcg_at_20
value: 86.634
- type: ndcg_at_100
value: 87.348
- type: ndcg_at_1000
value: 87.48599999999999
- type: recall_at_1
value: 80
- type: recall_at_3
value: 87
- type: recall_at_5
value: 89.333
- type: recall_at_10
value: 91.667
- type: recall_at_20
value: 95
- type: recall_at_100
value: 99
- type: recall_at_1000
value: 100
- type: main_score
value: 85.751
task:
type: Retrieval
- dataset:
config: default
name: MTEB LeCaRDv2 (default)
revision: b78e18688c3d012a33dc3676597c1d1b2243ce1c
split: test
type: mteb/LeCaRDv2
metrics:
- type: ndcg_at_1
value: 87.421
- type: ndcg_at_3
value: 83.159
- type: ndcg_at_5
value: 79.818
- type: ndcg_at_10
value: 74.168
- type: ndcg_at_20
value: 67.81
- type: ndcg_at_100
value: 80.432
- type: ndcg_at_1000
value: 84.423
- type: recall_at_1
value: 4.1450000000000005
- type: recall_at_3
value: 10.988000000000001
- type: recall_at_5
value: 16.808999999999997
- type: recall_at_10
value: 29.329
- type: recall_at_20
value: 48.425000000000004
- type: recall_at_100
value: 89.63600000000001
- type: recall_at_1000
value: 99.823
- type: main_score
value: 74.168
task:
type: Retrieval
- dataset:
config: default
name: MTEB LegalBenchConsumerContractsQA (default)
revision: b23590301ec94e8087e2850b21d43d4956b1cca9
split: test
type: mteb/legalbench_consumer_contracts_qa
metrics:
- type: ndcg_at_1
value: 73.485
- type: ndcg_at_3
value: 81.977
- type: ndcg_at_5
value: 84.63000000000001
- type: ndcg_at_10
value: 85.444
- type: ndcg_at_20
value: 86.008
- type: ndcg_at_100
value: 86.262
- type: ndcg_at_1000
value: 86.262
- type: recall_at_1
value: 73.485
- type: recall_at_3
value: 87.626
- type: recall_at_5
value: 93.939
- type: recall_at_10
value: 96.465
- type: recall_at_20
value: 98.737
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: main_score
value: 85.444
task:
type: Retrieval
- dataset:
config: default
name: MTEB LegalBenchCorporateLobbying (default)
revision: f69691c650464e62546d7f2a4536f8f87c891e38
split: test
type: mteb/legalbench_corporate_lobbying
metrics:
- type: ndcg_at_1
value: 91.471
- type: ndcg_at_3
value: 95.84700000000001
- type: ndcg_at_5
value: 96.088
- type: ndcg_at_10
value: 96.17999999999999
- type: ndcg_at_20
value: 96.17999999999999
- type: ndcg_at_100
value: 96.17999999999999
- type: ndcg_at_1000
value: 96.259
- type: recall_at_1
value: 91.471
- type: recall_at_3
value: 98.529
- type: recall_at_5
value: 99.118
- type: recall_at_10
value: 99.412
- type: recall_at_20
value: 99.412
- type: recall_at_100
value: 99.412
- type: recall_at_1000
value: 100
- type: main_score
value: 96.17999999999999
task:
type: Retrieval
- dataset:
config: default
name: MTEB LegalQuAD (default)
revision: 37aa6cfb01d48960b0f8e3f17d6e3d99bf1ebc3e
split: test
type: mteb/LegalQuAD
metrics:
- type: ndcg_at_1
value: 48
- type: ndcg_at_3
value: 59.397999999999996
- type: ndcg_at_5
value: 61.05500000000001
- type: ndcg_at_10
value: 63.219
- type: ndcg_at_20
value: 65.102
- type: ndcg_at_100
value: 67.254
- type: ndcg_at_1000
value: 67.746
- type: recall_at_1
value: 48
- type: recall_at_3
value: 67
- type: recall_at_5
value: 71
- type: recall_at_10
value: 77.5
- type: recall_at_20
value: 85
- type: recall_at_100
value: 96.5
- type: recall_at_1000
value: 100
- type: main_score
value: 63.219
task:
type: Retrieval
- dataset:
config: default
name: MTEB LegalSummarization (default)
revision: 3bb1a05c66872889662af04c5691c14489cebd72
split: test
type: mteb/legal_summarization
metrics:
- type: ndcg_at_1
value: 58.451
- type: ndcg_at_3
value: 63.70099999999999
- type: ndcg_at_5
value: 66.792
- type: ndcg_at_10
value: 69.76
- type: ndcg_at_20
value: 71.487
- type: ndcg_at_100
value: 73.6
- type: ndcg_at_1000
value: 74.05000000000001
- type: recall_at_1
value: 52.028
- type: recall_at_3
value: 66.7
- type: recall_at_5
value: 74.119
- type: recall_at_10
value: 82.595
- type: recall_at_20
value: 88.209
- type: recall_at_100
value: 97.24000000000001
- type: recall_at_1000
value: 100
- type: main_score
value: 69.76
task:
type: Retrieval
- dataset:
config: ar
name: MTEB MintakaRetrieval (ar)
revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e
split: test
type: jinaai/mintakaqa
metrics:
- type: ndcg_at_1
value: 19.791
- type: ndcg_at_3
value: 29.751
- type: ndcg_at_5
value: 32.83
- type: ndcg_at_10
value: 35.553000000000004
- type: ndcg_at_20
value: 37.528
- type: ndcg_at_100
value: 40.025
- type: ndcg_at_1000
value: 42.693
- type: recall_at_1
value: 19.791
- type: recall_at_3
value: 36.632
- type: recall_at_5
value: 44.076
- type: recall_at_10
value: 52.474
- type: recall_at_20
value: 60.281
- type: recall_at_100
value: 73.94500000000001
- type: recall_at_1000
value: 96.096
- type: main_score
value: 35.553000000000004
task:
type: Retrieval
- dataset:
config: de
name: MTEB MintakaRetrieval (de)
revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e
split: test
type: jinaai/mintakaqa
metrics:
- type: ndcg_at_1
value: 34.288000000000004
- type: ndcg_at_3
value: 47.29
- type: ndcg_at_5
value: 50.622
- type: ndcg_at_10
value: 53.291999999999994
- type: ndcg_at_20
value: 55.062999999999995
- type: ndcg_at_100
value: 56.987
- type: ndcg_at_1000
value: 58.084
- type: recall_at_1
value: 34.288000000000004
- type: recall_at_3
value: 56.486999999999995
- type: recall_at_5
value: 64.532
- type: recall_at_10
value: 72.746
- type: recall_at_20
value: 79.697
- type: recall_at_100
value: 90.185
- type: recall_at_1000
value: 98.989
- type: main_score
value: 53.291999999999994
task:
type: Retrieval
- dataset:
config: es
name: MTEB MintakaRetrieval (es)
revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e
split: test
type: jinaai/mintakaqa
metrics:
- type: ndcg_at_1
value: 31.889
- type: ndcg_at_3
value: 45.182
- type: ndcg_at_5
value: 48.475
- type: ndcg_at_10
value: 51.402
- type: ndcg_at_20
value: 53.089
- type: ndcg_at_100
value: 55.116
- type: ndcg_at_1000
value: 56.333999999999996
- type: recall_at_1
value: 31.889
- type: recall_at_3
value: 54.455
- type: recall_at_5
value: 62.417
- type: recall_at_10
value: 71.328
- type: recall_at_20
value: 77.97
- type: recall_at_100
value: 88.944
- type: recall_at_1000
value: 98.639
- type: main_score
value: 51.402
task:
type: Retrieval
- dataset:
config: fr
name: MTEB MintakaRetrieval (fr)
revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e
split: test
type: jinaai/mintakaqa
metrics:
- type: ndcg_at_1
value: 32.555
- type: ndcg_at_3
value: 45.278
- type: ndcg_at_5
value: 48.559000000000005
- type: ndcg_at_10
value: 51.485
- type: ndcg_at_20
value: 53.263000000000005
- type: ndcg_at_100
value: 55.221
- type: ndcg_at_1000
value: 56.501999999999995
- type: recall_at_1
value: 32.555
- type: recall_at_3
value: 54.054
- type: recall_at_5
value: 62.039
- type: recall_at_10
value: 70.966
- type: recall_at_20
value: 77.969
- type: recall_at_100
value: 88.411
- type: recall_at_1000
value: 98.69
- type: main_score
value: 51.485
task:
type: Retrieval
- dataset:
config: hi
name: MTEB MintakaRetrieval (hi)
revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e
split: test
type: jinaai/mintakaqa
metrics:
- type: ndcg_at_1
value: 24.757
- type: ndcg_at_3
value: 35.427
- type: ndcg_at_5
value: 38.431
- type: ndcg_at_10
value: 41.459
- type: ndcg_at_20
value: 44.137
- type: ndcg_at_100
value: 47.174
- type: ndcg_at_1000
value: 48.907000000000004
- type: recall_at_1
value: 24.757
- type: recall_at_3
value: 43.082
- type: recall_at_5
value: 50.336999999999996
- type: recall_at_10
value: 59.611000000000004
- type: recall_at_20
value: 70.157
- type: recall_at_100
value: 86.387
- type: recall_at_1000
value: 100
- type: main_score
value: 41.459
task:
type: Retrieval
- dataset:
config: it
name: MTEB MintakaRetrieval (it)
revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e
split: test
type: jinaai/mintakaqa
metrics:
- type: ndcg_at_1
value: 32.818000000000005
- type: ndcg_at_3
value: 46.503
- type: ndcg_at_5
value: 49.68
- type: ndcg_at_10
value: 52.510999999999996
- type: ndcg_at_20
value: 54.269999999999996
- type: ndcg_at_100
value: 56.17100000000001
- type: ndcg_at_1000
value: 57.38100000000001
- type: recall_at_1
value: 32.818000000000005
- type: recall_at_3
value: 56.033
- type: recall_at_5
value: 63.715999999999994
- type: recall_at_10
value: 72.48400000000001
- type: recall_at_20
value: 79.374
- type: recall_at_100
value: 89.436
- type: recall_at_1000
value: 98.914
- type: main_score
value: 52.510999999999996
task:
type: Retrieval
- dataset:
config: ja
name: MTEB MintakaRetrieval (ja)
revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e
split: test
type: jinaai/mintakaqa
metrics:
- type: ndcg_at_1
value: 25.216
- type: ndcg_at_3
value: 35.982
- type: ndcg_at_5
value: 38.694
- type: ndcg_at_10
value: 41.585
- type: ndcg_at_20
value: 43.334
- type: ndcg_at_100
value: 45.831
- type: ndcg_at_1000
value: 48.06
- type: recall_at_1
value: 25.216
- type: recall_at_3
value: 43.599
- type: recall_at_5
value: 50.173
- type: recall_at_10
value: 59.083
- type: recall_at_20
value: 65.96
- type: recall_at_100
value: 79.542
- type: recall_at_1000
value: 97.794
- type: main_score
value: 41.585
task:
type: Retrieval
- dataset:
config: pt
name: MTEB MintakaRetrieval (pt)
revision: efa78cc2f74bbcd21eff2261f9e13aebe40b814e
split: test
type: jinaai/mintakaqa
metrics:
- type: ndcg_at_1
value: 33.517
- type: ndcg_at_3
value: 46.955999999999996
- type: ndcg_at_5
value: 50.441
- type: ndcg_at_10
value: 53.256
- type: ndcg_at_20
value: 55.086
- type: ndcg_at_100
value: 57.104
- type: ndcg_at_1000
value: 58.07600000000001
- type: recall_at_1
value: 33.517
- type: recall_at_3
value: 56.245
- type: recall_at_5
value: 64.63499999999999
- type: recall_at_10
value: 73.258
- type: recall_at_20
value: 80.47999999999999
- type: recall_at_100
value: 91.27
- type: recall_at_1000
value: 99.10799999999999
- type: main_score
value: 53.256
task:
type: Retrieval
- dataset:
config: default
name: MTEB NFCorpus (default)
revision: ec0fa4fe99da2ff19ca1214b7966684033a58814
split: test
type: mteb/nfcorpus
metrics:
- type: ndcg_at_1
value: 51.702999999999996
- type: ndcg_at_3
value: 48.064
- type: ndcg_at_5
value: 46.379
- type: ndcg_at_10
value: 43.663999999999994
- type: ndcg_at_20
value: 41.407
- type: ndcg_at_100
value: 42.083
- type: ndcg_at_1000
value: 52.335
- type: recall_at_1
value: 6.241
- type: recall_at_3
value: 12.214
- type: recall_at_5
value: 16.473
- type: recall_at_10
value: 21.84
- type: recall_at_20
value: 27.474999999999998
- type: recall_at_100
value: 45.01
- type: recall_at_1000
value: 80.71300000000001
- type: main_score
value: 43.663999999999994
task:
type: Retrieval
- dataset:
config: default
name: MTEB NFCorpus-PL (default)
revision: 9a6f9567fda928260afed2de480d79c98bf0bec0
split: test
type: clarin-knext/nfcorpus-pl
metrics:
- type: ndcg_at_1
value: 41.641
- type: ndcg_at_3
value: 37.617
- type: ndcg_at_5
value: 36.024
- type: ndcg_at_10
value: 33.51
- type: ndcg_at_20
value: 31.575999999999997
- type: ndcg_at_100
value: 31.601000000000003
- type: ndcg_at_1000
value: 41.099000000000004
- type: recall_at_1
value: 4.61
- type: recall_at_3
value: 9.366
- type: recall_at_5
value: 11.793
- type: recall_at_10
value: 16.255
- type: recall_at_20
value: 20.713
- type: recall_at_100
value: 33.396
- type: recall_at_1000
value: 65.532
- type: main_score
value: 33.51
task:
type: Retrieval
- dataset:
config: default
name: MTEB NQ (default)
revision: b774495ed302d8c44a3a7ea25c90dbce03968f31
split: test
type: mteb/nq
metrics:
- type: ndcg_at_1
value: 50.753
- type: ndcg_at_3
value: 62.541000000000004
- type: ndcg_at_5
value: 66.46600000000001
- type: ndcg_at_10
value: 69.65400000000001
- type: ndcg_at_20
value: 70.91499999999999
- type: ndcg_at_100
value: 71.908
- type: ndcg_at_1000
value: 72.08200000000001
- type: recall_at_1
value: 45.293
- type: recall_at_3
value: 71.089
- type: recall_at_5
value: 79.93
- type: recall_at_10
value: 89.01599999999999
- type: recall_at_20
value: 93.60300000000001
- type: recall_at_100
value: 98.501
- type: recall_at_1000
value: 99.768
- type: main_score
value: 69.65400000000001
task:
type: Retrieval
- dataset:
config: default
name: MTEB NQ-PL (default)
revision: f171245712cf85dd4700b06bef18001578d0ca8d
split: test
type: clarin-knext/nq-pl
metrics:
- type: ndcg_at_1
value: 34.791
- type: ndcg_at_3
value: 45.418
- type: ndcg_at_5
value: 49.486000000000004
- type: ndcg_at_10
value: 53.141000000000005
- type: ndcg_at_20
value: 55.230999999999995
- type: ndcg_at_100
value: 57.358
- type: ndcg_at_1000
value: 58.166
- type: recall_at_1
value: 31.04
- type: recall_at_3
value: 53.179
- type: recall_at_5
value: 62.539
- type: recall_at_10
value: 73.08099999999999
- type: recall_at_20
value: 80.83500000000001
- type: recall_at_100
value: 91.503
- type: recall_at_1000
value: 97.429
- type: main_score
value: 53.141000000000005
task:
type: Retrieval
- dataset:
config: default
name: MTEB Quora-PL (default)
revision: 0be27e93455051e531182b85e85e425aba12e9d4
split: validation
type: clarin-knext/quora-pl
metrics:
- type: ndcg_at_1
value: 76.99000000000001
- type: ndcg_at_3
value: 81.781
- type: ndcg_at_5
value: 83.627
- type: ndcg_at_10
value: 85.146
- type: ndcg_at_20
value: 86.015
- type: ndcg_at_100
value: 86.745
- type: ndcg_at_1000
value: 86.882
- type: recall_at_1
value: 66.806
- type: recall_at_3
value: 84.09400000000001
- type: recall_at_5
value: 89.09899999999999
- type: recall_at_10
value: 93.512
- type: recall_at_20
value: 96.365
- type: recall_at_100
value: 99.22
- type: recall_at_1000
value: 99.937
- type: main_score
value: 85.146
task:
type: Retrieval
- dataset:
config: default
name: MTEB QuoraRetrieval (default)
revision: e4e08e0b7dbe3c8700f0daef558ff32256715259
split: test
type: mteb/quora
metrics:
- type: ndcg_at_1
value: 83.66
- type: ndcg_at_3
value: 87.863
- type: ndcg_at_5
value: 89.279
- type: ndcg_at_10
value: 90.372
- type: ndcg_at_20
value: 90.955
- type: ndcg_at_100
value: 91.352
- type: ndcg_at_1000
value: 91.39500000000001
- type: recall_at_1
value: 72.75399999999999
- type: recall_at_3
value: 89.41799999999999
- type: recall_at_5
value: 93.509
- type: recall_at_10
value: 96.679
- type: recall_at_20
value: 98.519
- type: recall_at_100
value: 99.845
- type: recall_at_1000
value: 99.998
- type: main_score
value: 90.372
task:
type: Retrieval
- dataset:
config: default
name: MTEB RiaNewsRetrieval (default)
revision: 82374b0bbacda6114f39ff9c5b925fa1512ca5d7
split: test
type: ai-forever/ria-news-retrieval
metrics:
- type: ndcg_at_1
value: 75.41
- type: ndcg_at_3
value: 83.13000000000001
- type: ndcg_at_5
value: 84.313
- type: ndcg_at_10
value: 85.009
- type: ndcg_at_20
value: 85.436
- type: ndcg_at_100
value: 85.875
- type: ndcg_at_1000
value: 86.048
- type: recall_at_1
value: 75.41
- type: recall_at_3
value: 88.38000000000001
- type: recall_at_5
value: 91.23
- type: recall_at_10
value: 93.34
- type: recall_at_20
value: 95.02000000000001
- type: recall_at_100
value: 97.37
- type: recall_at_1000
value: 98.78
- type: main_score
value: 85.009
task:
type: Retrieval
- dataset:
config: default
name: MTEB RuBQRetrieval (default)
revision: e19b6ffa60b3bc248e0b41f4cc37c26a55c2a67b
split: test
type: ai-forever/rubq-retrieval
metrics:
- type: ndcg_at_1
value: 63.652
- type: ndcg_at_3
value: 67.829
- type: ndcg_at_5
value: 72.141
- type: ndcg_at_10
value: 75.551
- type: ndcg_at_20
value: 76.925
- type: ndcg_at_100
value: 77.813
- type: ndcg_at_1000
value: 77.994
- type: recall_at_1
value: 45.09
- type: recall_at_3
value: 71.562
- type: recall_at_5
value: 81.474
- type: recall_at_10
value: 90.237
- type: recall_at_20
value: 94.679
- type: recall_at_100
value: 98.752
- type: recall_at_1000
value: 99.83999999999999
- type: main_score
value: 75.551
task:
type: Retrieval
- dataset:
config: default
name: MTEB SCIDOCS (default)
revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88
split: test
type: mteb/scidocs
metrics:
- type: ndcg_at_1
value: 33.7
- type: ndcg_at_3
value: 28.360999999999997
- type: ndcg_at_5
value: 25.259999999999998
- type: ndcg_at_10
value: 30.775999999999996
- type: ndcg_at_20
value: 34.782000000000004
- type: ndcg_at_100
value: 41.753
- type: ndcg_at_1000
value: 46.887
- type: recall_at_1
value: 6.843000000000001
- type: recall_at_3
value: 16.228
- type: recall_at_5
value: 22.828
- type: recall_at_10
value: 33.007
- type: recall_at_20
value: 42.433
- type: recall_at_100
value: 64.967
- type: recall_at_1000
value: 89.587
- type: main_score
value: 30.775999999999996
task:
type: Retrieval
- dataset:
config: default
name: MTEB SCIDOCS-PL (default)
revision: 45452b03f05560207ef19149545f168e596c9337
split: test
type: clarin-knext/scidocs-pl
metrics:
- type: ndcg_at_1
value: 26.5
- type: ndcg_at_3
value: 21.079
- type: ndcg_at_5
value: 18.63
- type: ndcg_at_10
value: 22.483
- type: ndcg_at_20
value: 25.552999999999997
- type: ndcg_at_100
value: 31.572
- type: ndcg_at_1000
value: 37.147000000000006
- type: recall_at_1
value: 5.367999999999999
- type: recall_at_3
value: 11.907
- type: recall_at_5
value: 16.631999999999998
- type: recall_at_10
value: 23.647000000000002
- type: recall_at_20
value: 30.857
- type: recall_at_100
value: 50.236999999999995
- type: recall_at_1000
value: 77.445
- type: main_score
value: 22.483
task:
type: Retrieval
- dataset:
config: default
name: MTEB SciFact (default)
revision: 0228b52cf27578f30900b9e5271d331663a030d7
split: test
type: mteb/scifact
metrics:
- type: ndcg_at_1
value: 74.333
- type: ndcg_at_3
value: 82.071
- type: ndcg_at_5
value: 83.83800000000001
- type: ndcg_at_10
value: 85.399
- type: ndcg_at_20
value: 85.57900000000001
- type: ndcg_at_100
value: 86.075
- type: ndcg_at_1000
value: 86.164
- type: recall_at_1
value: 70.994
- type: recall_at_3
value: 87.417
- type: recall_at_5
value: 91.89399999999999
- type: recall_at_10
value: 96.167
- type: recall_at_20
value: 96.833
- type: recall_at_100
value: 99.333
- type: recall_at_1000
value: 100
- type: main_score
value: 85.399
task:
type: Retrieval
- dataset:
config: default
name: MTEB SciFact-PL (default)
revision: 47932a35f045ef8ed01ba82bf9ff67f6e109207e
split: test
type: clarin-knext/scifact-pl
metrics:
- type: ndcg_at_1
value: 65.333
- type: ndcg_at_3
value: 73.291
- type: ndcg_at_5
value: 75.149
- type: ndcg_at_10
value: 77.633
- type: ndcg_at_20
value: 78.236
- type: ndcg_at_100
value: 79.182
- type: ndcg_at_1000
value: 79.431
- type: recall_at_1
value: 61.99400000000001
- type: recall_at_3
value: 79.01700000000001
- type: recall_at_5
value: 83.72800000000001
- type: recall_at_10
value: 90.72200000000001
- type: recall_at_20
value: 93
- type: recall_at_100
value: 98
- type: recall_at_1000
value: 100
- type: main_score
value: 77.633
task:
type: Retrieval
- dataset:
config: default
name: MTEB StackOverflowQA (default)
revision: db8f169f3894c14a00251061f957b2063eef2bd5
split: test
type: CoIR-Retrieval/stackoverflow-qa
metrics:
- type: ndcg_at_1
value: 90.07
- type: ndcg_at_3
value: 93.30199999999999
- type: ndcg_at_5
value: 93.812
- type: ndcg_at_10
value: 94.219
- type: ndcg_at_20
value: 94.46799999999999
- type: ndcg_at_100
value: 94.581
- type: ndcg_at_1000
value: 94.626
- type: recall_at_1
value: 90.07
- type: recall_at_3
value: 95.537
- type: recall_at_5
value: 96.78999999999999
- type: recall_at_10
value: 98.044
- type: recall_at_20
value: 99.047
- type: recall_at_100
value: 99.649
- type: recall_at_1000
value: 100
- type: main_score
value: 94.219
task:
type: Retrieval
- dataset:
config: default
name: MTEB SyntecRetrieval (default)
revision: 19661ccdca4dfc2d15122d776b61685f48c68ca9
split: test
type: lyon-nlp/mteb-fr-retrieval-syntec-s2p
metrics:
- type: ndcg_at_1
value: 83
- type: ndcg_at_3
value: 90.809
- type: ndcg_at_5
value: 91.583
- type: ndcg_at_10
value: 92.199
- type: ndcg_at_20
value: 92.199
- type: ndcg_at_100
value: 92.199
- type: ndcg_at_1000
value: 92.199
- type: recall_at_1
value: 83
- type: recall_at_3
value: 96
- type: recall_at_5
value: 98
- type: recall_at_10
value: 100
- type: recall_at_20
value: 100
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: main_score
value: 92.199
task:
type: Retrieval
- dataset:
config: default
name: MTEB SyntheticText2SQL (default)
revision: 686b87296c3a0191b5d9415a00526c62db9fce09
split: test
type: CoIR-Retrieval/synthetic-text2sql
metrics:
- type: ndcg_at_1
value: 20.526
- type: ndcg_at_3
value: 60.12
- type: ndcg_at_5
value: 62.134
- type: ndcg_at_10
value: 63.50599999999999
- type: ndcg_at_20
value: 64.167
- type: ndcg_at_100
value: 64.687
- type: ndcg_at_1000
value: 64.801
- type: recall_at_1
value: 20.526
- type: recall_at_3
value: 84.721
- type: recall_at_5
value: 89.574
- type: recall_at_10
value: 93.762
- type: recall_at_20
value: 96.36
- type: recall_at_100
value: 99.09400000000001
- type: recall_at_1000
value: 99.966
- type: main_score
value: 63.50599999999999
task:
type: Retrieval
- dataset:
config: default
name: MTEB TRECCOVID (default)
revision: bb9466bac8153a0349341eb1b22e06409e78ef4e
split: test
type: mteb/trec-covid
metrics:
- type: ndcg_at_1
value: 76
- type: ndcg_at_3
value: 78.899
- type: ndcg_at_5
value: 78.212
- type: ndcg_at_10
value: 75.09700000000001
- type: ndcg_at_20
value: 72.158
- type: ndcg_at_100
value: 58.465999999999994
- type: ndcg_at_1000
value: 53.702000000000005
- type: recall_at_1
value: 0.231
- type: recall_at_3
value: 0.7000000000000001
- type: recall_at_5
value: 1.146
- type: recall_at_10
value: 2.174
- type: recall_at_20
value: 4.031
- type: recall_at_100
value: 14.713999999999999
- type: recall_at_1000
value: 50.8
- type: main_score
value: 75.09700000000001
task:
type: Retrieval
- dataset:
config: default
name: MTEB TRECCOVID-PL (default)
revision: 81bcb408f33366c2a20ac54adafad1ae7e877fdd
split: test
type: clarin-knext/trec-covid-pl
metrics:
- type: ndcg_at_1
value: 75
- type: ndcg_at_3
value: 75.531
- type: ndcg_at_5
value: 75.327
- type: ndcg_at_10
value: 74.28
- type: ndcg_at_20
value: 71.5
- type: ndcg_at_100
value: 58.412
- type: ndcg_at_1000
value: 52.580000000000005
- type: recall_at_1
value: 0.214
- type: recall_at_3
value: 0.647
- type: recall_at_5
value: 1.083
- type: recall_at_10
value: 2.141
- type: recall_at_20
value: 3.9309999999999996
- type: recall_at_100
value: 14.738999999999999
- type: recall_at_1000
value: 49.494
- type: main_score
value: 74.28
task:
type: Retrieval
- dataset:
config: default
name: MTEB Touche2020 (default)
revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f
split: test
type: mteb/touche2020
metrics:
- type: ndcg_at_1
value: 20.408
- type: ndcg_at_3
value: 23.368
- type: ndcg_at_5
value: 24.795
- type: ndcg_at_10
value: 24.442
- type: ndcg_at_20
value: 26.712000000000003
- type: ndcg_at_100
value: 38.218999999999994
- type: ndcg_at_1000
value: 50.395
- type: recall_at_1
value: 2.414
- type: recall_at_3
value: 6.3549999999999995
- type: recall_at_5
value: 9.888
- type: recall_at_10
value: 16.31
- type: recall_at_20
value: 25.369000000000003
- type: recall_at_100
value: 51.449999999999996
- type: recall_at_1000
value: 88.532
- type: main_score
value: 24.442
task:
type: Retrieval
- dataset:
config: default
name: MTEB ARCChallenge (default)
revision: c481e0da3dcbbad8bce7721dea9085b74320a0a3
split: test
type: RAR-b/ARC-Challenge
metrics:
- type: ndcg_at_1
value: 8.959
- type: ndcg_at_3
value: 16.238
- type: ndcg_at_5
value: 18.841
- type: ndcg_at_10
value: 21.606
- type: ndcg_at_20
value: 24.326
- type: ndcg_at_100
value: 28.410999999999998
- type: ndcg_at_1000
value: 31.279
- type: recall_at_1
value: 8.959
- type: recall_at_3
value: 21.416
- type: recall_at_5
value: 27.73
- type: recall_at_10
value: 36.348
- type: recall_at_20
value: 47.184
- type: recall_at_100
value: 69.539
- type: recall_at_1000
value: 92.747
- type: main_score
value: 21.606
task:
type: Retrieval
- dataset:
config: default
name: MTEB AlphaNLI (default)
revision: 303f40ef3d50918d3dc43577d33f2f7344ad72c1
split: test
type: RAR-b/alphanli
metrics:
- type: ndcg_at_1
value: 29.047
- type: ndcg_at_3
value: 37.782
- type: ndcg_at_5
value: 39.989999999999995
- type: ndcg_at_10
value: 41.926
- type: ndcg_at_20
value: 43.573
- type: ndcg_at_100
value: 45.957
- type: ndcg_at_1000
value: 47.799
- type: recall_at_1
value: 29.047
- type: recall_at_3
value: 43.799
- type: recall_at_5
value: 49.151
- type: recall_at_10
value: 55.222
- type: recall_at_20
value: 61.748999999999995
- type: recall_at_100
value: 74.543
- type: recall_at_1000
value: 89.491
- type: main_score
value: 41.926
task:
type: Retrieval
- dataset:
config: default
name: MTEB BSARDRetrieval (default)
revision: 5effa1b9b5fa3b0f9e12523e6e43e5f86a6e6d59
split: test
type: maastrichtlawtech/bsard
metrics:
- type: ndcg_at_1
value: 15.315000000000001
- type: ndcg_at_3
value: 22.742
- type: ndcg_at_5
value: 25.146
- type: ndcg_at_10
value: 28.993000000000002
- type: ndcg_at_20
value: 30.797
- type: ndcg_at_100
value: 34.189
- type: ndcg_at_1000
value: 36.507
- type: recall_at_1
value: 15.315000000000001
- type: recall_at_3
value: 27.927999999999997
- type: recall_at_5
value: 33.784
- type: recall_at_10
value: 45.495000000000005
- type: recall_at_20
value: 52.703
- type: recall_at_100
value: 71.622
- type: recall_at_1000
value: 90.54100000000001
- type: main_score
value: 71.622
task:
type: Retrieval
- dataset:
config: default
name: MTEB ClimateFEVER (default)
revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380
split: test
type: mteb/climate-fever
metrics:
- type: ndcg_at_1
value: 38.111
- type: ndcg_at_3
value: 34.489999999999995
- type: ndcg_at_5
value: 36.986999999999995
- type: ndcg_at_10
value: 41.825
- type: ndcg_at_20
value: 45.326
- type: ndcg_at_100
value: 50.207
- type: ndcg_at_1000
value: 52.686
- type: recall_at_1
value: 16.898
- type: recall_at_3
value: 31.636999999999997
- type: recall_at_5
value: 39.147
- type: recall_at_10
value: 49.787
- type: recall_at_20
value: 59.41499999999999
- type: recall_at_100
value: 77.506
- type: recall_at_1000
value: 90.803
- type: main_score
value: 41.825
task:
type: Retrieval
- dataset:
config: default
name: MTEB DBPedia-PL (default)
revision: 76afe41d9af165cc40999fcaa92312b8b012064a
split: test
type: clarin-knext/dbpedia-pl
metrics:
- type: ndcg_at_1
value: 50.875
- type: ndcg_at_3
value: 43.745
- type: ndcg_at_5
value: 42.186
- type: ndcg_at_10
value: 40.506
- type: ndcg_at_20
value: 40.372
- type: ndcg_at_100
value: 45.967
- type: ndcg_at_1000
value: 53.247
- type: recall_at_1
value: 8.14
- type: recall_at_3
value: 14.038
- type: recall_at_5
value: 18.394
- type: recall_at_10
value: 24.476
- type: recall_at_20
value: 32.141999999999996
- type: recall_at_100
value: 53.027
- type: recall_at_1000
value: 76.108
- type: main_score
value: 40.506
task:
type: Retrieval
- dataset:
config: default
name: MTEB FEVER (default)
revision: bea83ef9e8fb933d90a2f1d5515737465d613e12
split: test
type: mteb/fever
metrics:
- type: ndcg_at_1
value: 91.899
- type: ndcg_at_3
value: 93.267
- type: ndcg_at_5
value: 93.757
- type: ndcg_at_10
value: 94.146
- type: ndcg_at_20
value: 94.42399999999999
- type: ndcg_at_100
value: 94.647
- type: ndcg_at_1000
value: 94.765
- type: recall_at_1
value: 85.329
- type: recall_at_3
value: 94.89
- type: recall_at_5
value: 96.185
- type: recall_at_10
value: 97.234
- type: recall_at_20
value: 98.059
- type: recall_at_100
value: 98.946
- type: recall_at_1000
value: 99.605
- type: main_score
value: 94.146
task:
type: Retrieval
- dataset:
config: default
name: MTEB GermanDPR (default)
revision: 5129d02422a66be600ac89cd3e8531b4f97d347d
split: test
type: deepset/germandpr
metrics:
- type: ndcg_at_1
value: 67.415
- type: ndcg_at_3
value: 81.684
- type: ndcg_at_5
value: 83.829
- type: ndcg_at_10
value: 84.624
- type: ndcg_at_20
value: 84.77900000000001
- type: ndcg_at_100
value: 84.832
- type: ndcg_at_1000
value: 84.832
- type: recall_at_1
value: 67.415
- type: recall_at_3
value: 91.61
- type: recall_at_5
value: 96.78
- type: recall_at_10
value: 99.122
- type: recall_at_20
value: 99.70700000000001
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: main_score
value: 84.624
task:
type: Retrieval
- dataset:
config: default
name: MTEB GermanQuAD-Retrieval (default)
revision: f5c87ae5a2e7a5106606314eef45255f03151bb3
split: test
type: mteb/germanquad-retrieval
metrics:
- type: ndcg_at_1
value: 92.967
- type: ndcg_at_3
value: 96.289
- type: ndcg_at_5
value: 96.626
- type: ndcg_at_10
value: 96.68900000000001
- type: ndcg_at_20
value: 96.767
- type: ndcg_at_100
value: 96.812
- type: ndcg_at_1000
value: 96.812
- type: recall_at_1
value: 92.967
- type: recall_at_3
value: 98.457
- type: recall_at_5
value: 99.274
- type: recall_at_10
value: 99.456
- type: recall_at_20
value: 99.773
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: main_score
value: 95.7191
task:
type: Retrieval
- dataset:
config: default
name: MTEB HellaSwag (default)
revision: a5c990205e017d10761197ccab3000936689c3ae
split: test
type: RAR-b/hellaswag
metrics:
- type: ndcg_at_1
value: 24.139
- type: ndcg_at_3
value: 34.455999999999996
- type: ndcg_at_5
value: 37.217
- type: ndcg_at_10
value: 39.655
- type: ndcg_at_20
value: 41.177
- type: ndcg_at_100
value: 43.695
- type: ndcg_at_1000
value: 45.528
- type: recall_at_1
value: 24.139
- type: recall_at_3
value: 41.894
- type: recall_at_5
value: 48.565999999999995
- type: recall_at_10
value: 56.065
- type: recall_at_20
value: 62.07899999999999
- type: recall_at_100
value: 75.812
- type: recall_at_1000
value: 90.5
- type: main_score
value: 39.655
task:
type: Retrieval
- dataset:
config: default
name: MTEB HotpotQA-PL (default)
revision: a0bd479ac97b4ccb5bd6ce320c415d0bb4beb907
split: test
type: clarin-knext/hotpotqa-pl
metrics:
- type: ndcg_at_1
value: 81.796
- type: ndcg_at_3
value: 68.66499999999999
- type: ndcg_at_5
value: 71.364
- type: ndcg_at_10
value: 73.414
- type: ndcg_at_20
value: 74.634
- type: ndcg_at_100
value: 76.276
- type: ndcg_at_1000
value: 77.34299999999999
- type: recall_at_1
value: 40.898
- type: recall_at_3
value: 66.009
- type: recall_at_5
value: 71.317
- type: recall_at_10
value: 76.435
- type: recall_at_20
value: 80.35799999999999
- type: recall_at_100
value: 87.54899999999999
- type: recall_at_1000
value: 94.537
- type: main_score
value: 73.414
task:
type: Retrieval
- dataset:
config: default
name: MTEB MSMARCO (default)
revision: c5a29a104738b98a9e76336939199e264163d4a0
split: dev
type: mteb/msmarco
metrics:
- type: ndcg_at_1
value: 23.854
- type: ndcg_at_3
value: 35.573
- type: ndcg_at_5
value: 39.96
- type: ndcg_at_10
value: 44.064
- type: ndcg_at_20
value: 46.572
- type: ndcg_at_100
value: 49.492000000000004
- type: ndcg_at_1000
value: 50.43
- type: recall_at_1
value: 23.202
- type: recall_at_3
value: 44.092999999999996
- type: recall_at_5
value: 54.6
- type: recall_at_10
value: 67.11399999999999
- type: recall_at_20
value: 76.79899999999999
- type: recall_at_100
value: 92.085
- type: recall_at_1000
value: 99.122
- type: main_score
value: 44.064
task:
type: Retrieval
- dataset:
config: default
name: MTEB PIQA (default)
revision: bb30be7e9184e6b6b1d99bbfe1bb90a3a81842e6
split: test
type: RAR-b/piqa
metrics:
- type: ndcg_at_1
value: 26.387
- type: ndcg_at_3
value: 36.972
- type: ndcg_at_5
value: 39.534000000000006
- type: ndcg_at_10
value: 42.443
- type: ndcg_at_20
value: 44.36
- type: ndcg_at_100
value: 46.575
- type: ndcg_at_1000
value: 48.024
- type: recall_at_1
value: 26.387
- type: recall_at_3
value: 44.45
- type: recall_at_5
value: 50.598
- type: recall_at_10
value: 59.57599999999999
- type: recall_at_20
value: 67.13799999999999
- type: recall_at_100
value: 79.217
- type: recall_at_1000
value: 91.023
- type: main_score
value: 42.443
task:
type: Retrieval
- dataset:
config: default
name: MTEB Quail (default)
revision: 1851bc536f8bdab29e03e29191c4586b1d8d7c5a
split: test
type: RAR-b/quail
metrics:
- type: ndcg_at_1
value: 7.242999999999999
- type: ndcg_at_3
value: 11.727
- type: ndcg_at_5
value: 13.69
- type: ndcg_at_10
value: 16.186
- type: ndcg_at_20
value: 17.988
- type: ndcg_at_100
value: 20.926000000000002
- type: ndcg_at_1000
value: 23.980999999999998
- type: recall_at_1
value: 7.242999999999999
- type: recall_at_3
value: 15.037
- type: recall_at_5
value: 19.853
- type: recall_at_10
value: 27.573999999999998
- type: recall_at_20
value: 34.669
- type: recall_at_100
value: 50.662
- type: recall_at_1000
value: 75.735
- type: main_score
value: 16.186
task:
type: Retrieval
- dataset:
config: default
name: MTEB RARbCode (default)
revision: 25f7d11a7ac12dcbb8d3836eb2de682b98c825e4
split: test
type: RAR-b/humanevalpack-mbpp-pooled
metrics:
- type: ndcg_at_1
value: 75.40400000000001
- type: ndcg_at_3
value: 84.796
- type: ndcg_at_5
value: 86.68599999999999
- type: ndcg_at_10
value: 87.63499999999999
- type: ndcg_at_20
value: 87.813
- type: ndcg_at_100
value: 87.912
- type: ndcg_at_1000
value: 87.938
- type: recall_at_1
value: 75.40400000000001
- type: recall_at_3
value: 91.24
- type: recall_at_5
value: 95.822
- type: recall_at_10
value: 98.585
- type: recall_at_20
value: 99.259
- type: recall_at_100
value: 99.798
- type: recall_at_1000
value: 100
- type: main_score
value: 87.63499999999999
task:
type: Retrieval
- dataset:
config: default
name: MTEB RARbMath (default)
revision: 2393603c0221ff52f448d12dd75f0856103c6cca
split: test
type: RAR-b/math-pooled
metrics:
- type: ndcg_at_1
value: 90.869
- type: ndcg_at_3
value: 92.971
- type: ndcg_at_5
value: 93.365
- type: ndcg_at_10
value: 93.75099999999999
- type: ndcg_at_20
value: 94.05799999999999
- type: ndcg_at_100
value: 94.426
- type: ndcg_at_1000
value: 94.46600000000001
- type: recall_at_1
value: 90.869
- type: recall_at_3
value: 94.414
- type: recall_at_5
value: 95.363
- type: recall_at_10
value: 96.55
- type: recall_at_20
value: 97.753
- type: recall_at_100
value: 99.699
- type: recall_at_1000
value: 100
- type: main_score
value: 93.75099999999999
task:
type: Retrieval
- dataset:
config: default
name: MTEB SIQA (default)
revision: 4ed8415e9dc24060deefc84be59e2db0aacbadcc
split: test
type: RAR-b/siqa
metrics:
- type: ndcg_at_1
value: 2.661
- type: ndcg_at_3
value: 4.207000000000001
- type: ndcg_at_5
value: 4.577
- type: ndcg_at_10
value: 5.219
- type: ndcg_at_20
value: 5.917
- type: ndcg_at_100
value: 7.9670000000000005
- type: ndcg_at_1000
value: 11.527999999999999
- type: recall_at_1
value: 2.661
- type: recall_at_3
value: 5.271
- type: recall_at_5
value: 6.192
- type: recall_at_10
value: 8.187999999999999
- type: recall_at_20
value: 10.952
- type: recall_at_100
value: 22.262
- type: recall_at_1000
value: 52.098
- type: main_score
value: 5.219
task:
type: Retrieval
- dataset:
config: default
name: MTEB SpartQA (default)
revision: 9ab3ca3ccdd0d43f9cd6d346a363935d127f4f45
split: test
type: RAR-b/spartqa
metrics:
- type: ndcg_at_1
value: 1.252
- type: ndcg_at_3
value: 3.644
- type: ndcg_at_5
value: 5.27
- type: ndcg_at_10
value: 7.768
- type: ndcg_at_20
value: 10.181
- type: ndcg_at_100
value: 14.29
- type: ndcg_at_1000
value: 18.417
- type: recall_at_1
value: 0.788
- type: recall_at_3
value: 5.157
- type: recall_at_5
value: 8.728
- type: recall_at_10
value: 15.786
- type: recall_at_20
value: 24.365000000000002
- type: recall_at_100
value: 43.553999999999995
- type: recall_at_1000
value: 73.66
- type: main_score
value: 7.768
task:
type: Retrieval
- dataset:
config: default
name: MTEB TempReasonL1 (default)
revision: 9097e99aa8c9d827189c65f2e11bfe756af439f6
split: test
type: RAR-b/TempReason-l1
metrics:
- type: ndcg_at_1
value: 0.1
- type: ndcg_at_3
value: 0.716
- type: ndcg_at_5
value: 1.095
- type: ndcg_at_10
value: 1.6889999999999998
- type: ndcg_at_20
value: 2.374
- type: ndcg_at_100
value: 4.125
- type: ndcg_at_1000
value: 9.126
- type: recall_at_1
value: 0.1
- type: recall_at_3
value: 1.175
- type: recall_at_5
value: 2.1
- type: recall_at_10
value: 3.975
- type: recall_at_20
value: 6.675000000000001
- type: recall_at_100
value: 16.575
- type: recall_at_1000
value: 59.275
- type: main_score
value: 1.6889999999999998
task:
type: Retrieval
- dataset:
config: default
name: MTEB TempReasonL2Fact (default)
revision: 13758bcf978613b249d0de4d0840f57815122bdf
split: test
type: RAR-b/TempReason-l2-fact
metrics:
- type: ndcg_at_1
value: 28.942
- type: ndcg_at_3
value: 45.412
- type: ndcg_at_5
value: 50.43299999999999
- type: ndcg_at_10
value: 53.976
- type: ndcg_at_20
value: 55.703
- type: ndcg_at_100
value: 57.445
- type: ndcg_at_1000
value: 57.838
- type: recall_at_1
value: 28.942
- type: recall_at_3
value: 57.495
- type: recall_at_5
value: 69.631
- type: recall_at_10
value: 80.452
- type: recall_at_20
value: 87.252
- type: recall_at_100
value: 96.44200000000001
- type: recall_at_1000
value: 99.518
- type: main_score
value: 53.976
task:
type: Retrieval
- dataset:
config: default
name: MTEB TempReasonL2Pure (default)
revision: 27668949b97bfb178901e0cf047cbee805305dc1
split: test
type: RAR-b/TempReason-l2-pure
metrics:
- type: ndcg_at_1
value: 2.001
- type: ndcg_at_3
value: 3.746
- type: ndcg_at_5
value: 4.665
- type: ndcg_at_10
value: 5.972
- type: ndcg_at_20
value: 7.321999999999999
- type: ndcg_at_100
value: 11.068
- type: ndcg_at_1000
value: 15.675
- type: recall_at_1
value: 2.001
- type: recall_at_3
value: 5.04
- type: recall_at_5
value: 7.3
- type: recall_at_10
value: 11.34
- type: recall_at_20
value: 16.713
- type: recall_at_100
value: 37.576
- type: recall_at_1000
value: 75.394
- type: main_score
value: 5.972
task:
type: Retrieval
- dataset:
config: default
name: MTEB TempReasonL3Fact (default)
revision: 4b70e90197901da24f3cfcd51d27111292878680
split: test
type: RAR-b/TempReason-l3-fact
metrics:
- type: ndcg_at_1
value: 19.114
- type: ndcg_at_3
value: 34.72
- type: ndcg_at_5
value: 40.509
- type: ndcg_at_10
value: 44.894
- type: ndcg_at_20
value: 47.021
- type: ndcg_at_100
value: 49.162
- type: ndcg_at_1000
value: 49.833
- type: recall_at_1
value: 19.114
- type: recall_at_3
value: 46.385
- type: recall_at_5
value: 60.438
- type: recall_at_10
value: 73.882
- type: recall_at_20
value: 82.219
- type: recall_at_100
value: 93.47
- type: recall_at_1000
value: 98.735
- type: main_score
value: 44.894
task:
type: Retrieval
- dataset:
config: default
name: MTEB TempReasonL3Pure (default)
revision: 68fba138e7e63daccecfbdad0a9d2714e56e34ff
split: test
type: RAR-b/TempReason-l3-pure
metrics:
- type: ndcg_at_1
value: 0.836
- type: ndcg_at_3
value: 5.319
- type: ndcg_at_5
value: 7.468
- type: ndcg_at_10
value: 10.282
- type: ndcg_at_20
value: 12.457
- type: ndcg_at_100
value: 16.384
- type: ndcg_at_1000
value: 20.081
- type: recall_at_1
value: 0.836
- type: recall_at_3
value: 8.744
- type: recall_at_5
value: 13.963000000000001
- type: recall_at_10
value: 22.729
- type: recall_at_20
value: 31.338
- type: recall_at_100
value: 52.824000000000005
- type: recall_at_1000
value: 82.784
- type: main_score
value: 10.282
task:
type: Retrieval
- dataset:
config: default
name: MTEB WinoGrande (default)
revision: f74c094f321077cf909ddfb8bccc1b5912a4ac28
split: test
type: RAR-b/winogrande
metrics:
- type: ndcg_at_1
value: 47.908
- type: ndcg_at_3
value: 71.58200000000001
- type: ndcg_at_5
value: 74.265
- type: ndcg_at_10
value: 75.61099999999999
- type: ndcg_at_20
value: 76.07300000000001
- type: ndcg_at_100
value: 76.249
- type: ndcg_at_1000
value: 76.249
- type: recall_at_1
value: 47.908
- type: recall_at_3
value: 86.74
- type: recall_at_5
value: 93.21199999999999
- type: recall_at_10
value: 97.316
- type: recall_at_20
value: 99.132
- type: recall_at_100
value: 100
- type: recall_at_1000
value: 100
- type: main_score
value: 75.61099999999999
task:
type: Retrieval
- dataset:
config: de
name: MTEB XMarket (de)
revision: dfe57acff5b62c23732a7b7d3e3fb84ff501708b
split: test
type: jinaai/xmarket_ml
metrics:
- type: ndcg_at_1
value: 30.394
- type: ndcg_at_3
value: 30.701
- type: ndcg_at_5
value: 31.574
- type: ndcg_at_10
value: 32.961
- type: ndcg_at_20
value: 34.765
- type: ndcg_at_100
value: 38.772
- type: ndcg_at_1000
value: 43.317
- type: recall_at_1
value: 10.193000000000001
- type: recall_at_3
value: 19.141
- type: recall_at_5
value: 24.362000000000002
- type: recall_at_10
value: 31.995
- type: recall_at_20
value: 40.047
- type: recall_at_100
value: 56.769000000000005
- type: recall_at_1000
value: 76.318
- type: main_score
value: 32.961
task:
type: Retrieval
- dataset:
config: en
name: MTEB XMarket (en)
revision: dfe57acff5b62c23732a7b7d3e3fb84ff501708b
split: test
type: jinaai/xmarket_ml
metrics:
- type: ndcg_at_1
value: 37.652
- type: ndcg_at_3
value: 38.444
- type: ndcg_at_5
value: 39.163
- type: ndcg_at_10
value: 40.557
- type: ndcg_at_20
value: 42.224000000000004
- type: ndcg_at_100
value: 46.817
- type: ndcg_at_1000
value: 51.939
- type: recall_at_1
value: 8.909
- type: recall_at_3
value: 18.673000000000002
- type: recall_at_5
value: 24.364
- type: recall_at_10
value: 32.919
- type: recall_at_20
value: 41.908
- type: recall_at_100
value: 61.663999999999994
- type: recall_at_1000
value: 80.619
- type: main_score
value: 40.557
task:
type: Retrieval
- dataset:
config: es
name: MTEB XMarket (es)
revision: dfe57acff5b62c23732a7b7d3e3fb84ff501708b
split: test
type: jinaai/xmarket_ml
metrics:
- type: ndcg_at_1
value: 32.168
- type: ndcg_at_3
value: 32.389
- type: ndcg_at_5
value: 33.054
- type: ndcg_at_10
value: 34.549
- type: ndcg_at_20
value: 36.34
- type: ndcg_at_100
value: 40.324
- type: ndcg_at_1000
value: 44.784
- type: recall_at_1
value: 10.845
- type: recall_at_3
value: 21.058
- type: recall_at_5
value: 26.327
- type: recall_at_10
value: 34.306
- type: recall_at_20
value: 42.46
- type: recall_at_100
value: 59.156
- type: recall_at_1000
value: 78.249
- type: main_score
value: 34.549
task:
type: Retrieval
- dataset:
config: ara-ara
name: MTEB XPQARetrieval (ara-ara)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 42
- type: ndcg_at_3
value: 43.802
- type: ndcg_at_5
value: 46.1
- type: ndcg_at_10
value: 50.858000000000004
- type: ndcg_at_20
value: 54.303999999999995
- type: ndcg_at_100
value: 57.692
- type: ndcg_at_1000
value: 58.97599999999999
- type: recall_at_1
value: 23.989
- type: recall_at_3
value: 42.753
- type: recall_at_5
value: 51.56699999999999
- type: recall_at_10
value: 63.92400000000001
- type: recall_at_20
value: 75.249
- type: recall_at_100
value: 90.851
- type: recall_at_1000
value: 99.733
- type: main_score
value: 50.858000000000004
task:
type: Retrieval
- dataset:
config: eng-ara
name: MTEB XPQARetrieval (eng-ara)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 31.2
- type: ndcg_at_3
value: 33.296
- type: ndcg_at_5
value: 35.727
- type: ndcg_at_10
value: 39.837
- type: ndcg_at_20
value: 43.354
- type: ndcg_at_100
value: 47.908
- type: ndcg_at_1000
value: 50.187000000000005
- type: recall_at_1
value: 18.007
- type: recall_at_3
value: 32.5
- type: recall_at_5
value: 41.422
- type: recall_at_10
value: 51.673
- type: recall_at_20
value: 63.144
- type: recall_at_100
value: 83.733
- type: recall_at_1000
value: 99.10900000000001
- type: main_score
value: 39.837
task:
type: Retrieval
- dataset:
config: ara-eng
name: MTEB XPQARetrieval (ara-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 40.431
- type: ndcg_at_3
value: 41.419
- type: ndcg_at_5
value: 44.051
- type: ndcg_at_10
value: 48.94
- type: ndcg_at_20
value: 52.532999999999994
- type: ndcg_at_100
value: 56.203
- type: ndcg_at_1000
value: 57.467999999999996
- type: recall_at_1
value: 22.534000000000002
- type: recall_at_3
value: 40.119
- type: recall_at_5
value: 49.569
- type: recall_at_10
value: 62.156
- type: recall_at_20
value: 74.191
- type: recall_at_100
value: 90.973
- type: recall_at_1000
value: 99.72999999999999
- type: main_score
value: 48.94
task:
type: Retrieval
- dataset:
config: deu-deu
name: MTEB XPQARetrieval (deu-deu)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 76.50099999999999
- type: ndcg_at_3
value: 79.38199999999999
- type: ndcg_at_5
value: 81.00500000000001
- type: ndcg_at_10
value: 82.786
- type: ndcg_at_20
value: 83.844
- type: ndcg_at_100
value: 84.708
- type: ndcg_at_1000
value: 84.956
- type: recall_at_1
value: 58.464000000000006
- type: recall_at_3
value: 79.963
- type: recall_at_5
value: 85.757
- type: recall_at_10
value: 90.372
- type: recall_at_20
value: 94.13
- type: recall_at_100
value: 98.24000000000001
- type: recall_at_1000
value: 100
- type: main_score
value: 82.786
task:
type: Retrieval
- dataset:
config: eng-deu
name: MTEB XPQARetrieval (eng-deu)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 52.611
- type: ndcg_at_3
value: 55.35099999999999
- type: ndcg_at_5
value: 57.452999999999996
- type: ndcg_at_10
value: 61.553999999999995
- type: ndcg_at_20
value: 63.919000000000004
- type: ndcg_at_100
value: 66.90700000000001
- type: ndcg_at_1000
value: 67.685
- type: recall_at_1
value: 33.47
- type: recall_at_3
value: 55.174
- type: recall_at_5
value: 63.512
- type: recall_at_10
value: 73.934
- type: recall_at_20
value: 81.26400000000001
- type: recall_at_100
value: 94.606
- type: recall_at_1000
value: 100
- type: main_score
value: 61.553999999999995
task:
type: Retrieval
- dataset:
config: deu-eng
name: MTEB XPQARetrieval (deu-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 70.235
- type: ndcg_at_3
value: 74.824
- type: ndcg_at_5
value: 76.47699999999999
- type: ndcg_at_10
value: 78.803
- type: ndcg_at_20
value: 80.19
- type: ndcg_at_100
value: 81.07799999999999
- type: ndcg_at_1000
value: 81.40899999999999
- type: recall_at_1
value: 52.818
- type: recall_at_3
value: 76.754
- type: recall_at_5
value: 82.637
- type: recall_at_10
value: 88.655
- type: recall_at_20
value: 93.61
- type: recall_at_100
value: 97.731
- type: recall_at_1000
value: 100
- type: main_score
value: 78.803
task:
type: Retrieval
- dataset:
config: spa-spa
name: MTEB XPQARetrieval (spa-spa)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 64.18700000000001
- type: ndcg_at_3
value: 62.714999999999996
- type: ndcg_at_5
value: 64.134
- type: ndcg_at_10
value: 68.143
- type: ndcg_at_20
value: 70.625
- type: ndcg_at_100
value: 73.333
- type: ndcg_at_1000
value: 74.02300000000001
- type: recall_at_1
value: 34.400999999999996
- type: recall_at_3
value: 57.654
- type: recall_at_5
value: 67.167
- type: recall_at_10
value: 76.31599999999999
- type: recall_at_20
value: 83.731
- type: recall_at_100
value: 95.502
- type: recall_at_1000
value: 99.58
- type: main_score
value: 68.143
task:
type: Retrieval
- dataset:
config: eng-spa
name: MTEB XPQARetrieval (eng-spa)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 47.667
- type: ndcg_at_3
value: 46.35
- type: ndcg_at_5
value: 47.879
- type: ndcg_at_10
value: 52.733
- type: ndcg_at_20
value: 55.620000000000005
- type: ndcg_at_100
value: 59.70100000000001
- type: ndcg_at_1000
value: 61.417
- type: recall_at_1
value: 23.394000000000002
- type: recall_at_3
value: 42.264
- type: recall_at_5
value: 51.144999999999996
- type: recall_at_10
value: 62.556
- type: recall_at_20
value: 71.269
- type: recall_at_100
value: 88.668
- type: recall_at_1000
value: 99.466
- type: main_score
value: 52.733
task:
type: Retrieval
- dataset:
config: spa-eng
name: MTEB XPQARetrieval (spa-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 61.285999999999994
- type: ndcg_at_3
value: 60.303
- type: ndcg_at_5
value: 62.062
- type: ndcg_at_10
value: 66.042
- type: ndcg_at_20
value: 68.509
- type: ndcg_at_100
value: 71.539
- type: ndcg_at_1000
value: 72.258
- type: recall_at_1
value: 32.224000000000004
- type: recall_at_3
value: 55.443
- type: recall_at_5
value: 65.67699999999999
- type: recall_at_10
value: 74.607
- type: recall_at_20
value: 82.234
- type: recall_at_100
value: 95.275
- type: recall_at_1000
value: 99.723
- type: main_score
value: 66.042
task:
type: Retrieval
- dataset:
config: fra-fra
name: MTEB XPQARetrieval (fr)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 71.429
- type: ndcg_at_3
value: 71.13000000000001
- type: ndcg_at_5
value: 72.709
- type: ndcg_at_10
value: 76.236
- type: ndcg_at_20
value: 77.78500000000001
- type: ndcg_at_100
value: 79.634
- type: ndcg_at_1000
value: 79.953
- type: recall_at_1
value: 45.943
- type: recall_at_3
value: 68.293
- type: recall_at_5
value: 76.5
- type: recall_at_10
value: 85.11999999999999
- type: recall_at_20
value: 90.069
- type: recall_at_100
value: 97.82600000000001
- type: recall_at_1000
value: 99.866
- type: main_score
value: 76.236
task:
type: Retrieval
- dataset:
config: eng-fra
name: MTEB XPQARetrieval (eng-fra)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 47.797
- type: ndcg_at_3
value: 49.514
- type: ndcg_at_5
value: 51.782
- type: ndcg_at_10
value: 55.891000000000005
- type: ndcg_at_20
value: 59.226
- type: ndcg_at_100
value: 62.612
- type: ndcg_at_1000
value: 63.749
- type: recall_at_1
value: 26.689
- type: recall_at_3
value: 47.408
- type: recall_at_5
value: 57.399
- type: recall_at_10
value: 67.147
- type: recall_at_20
value: 77.837
- type: recall_at_100
value: 92.494
- type: recall_at_1000
value: 99.74
- type: main_score
value: 55.891000000000005
task:
type: Retrieval
- dataset:
config: fra-eng
name: MTEB XPQARetrieval (fra-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 68.625
- type: ndcg_at_3
value: 68.239
- type: ndcg_at_5
value: 70.175
- type: ndcg_at_10
value: 73.452
- type: ndcg_at_20
value: 75.66000000000001
- type: ndcg_at_100
value: 77.506
- type: ndcg_at_1000
value: 77.936
- type: recall_at_1
value: 44.035999999999994
- type: recall_at_3
value: 65.291
- type: recall_at_5
value: 74.37899999999999
- type: recall_at_10
value: 82.15
- type: recall_at_20
value: 89.457
- type: recall_at_100
value: 97.194
- type: recall_at_1000
value: 99.933
- type: main_score
value: 73.452
task:
type: Retrieval
- dataset:
config: hin-hin
name: MTEB XPQARetrieval (hin-hin)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 66.703
- type: ndcg_at_3
value: 72.993
- type: ndcg_at_5
value: 75.138
- type: ndcg_at_10
value: 77.371
- type: ndcg_at_20
value: 78.389
- type: ndcg_at_100
value: 79.623
- type: ndcg_at_1000
value: 79.975
- type: recall_at_1
value: 57.094
- type: recall_at_3
value: 77.2
- type: recall_at_5
value: 82.50800000000001
- type: recall_at_10
value: 88.486
- type: recall_at_20
value: 91.863
- type: recall_at_100
value: 97.359
- type: recall_at_1000
value: 99.892
- type: main_score
value: 77.371
task:
type: Retrieval
- dataset:
config: eng-hin
name: MTEB XPQARetrieval (eng-hin)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 29.837999999999997
- type: ndcg_at_3
value: 34.187
- type: ndcg_at_5
value: 37.132
- type: ndcg_at_10
value: 41.357
- type: ndcg_at_20
value: 44.522
- type: ndcg_at_100
value: 49.486999999999995
- type: ndcg_at_1000
value: 51.458000000000006
- type: recall_at_1
value: 24.959999999999997
- type: recall_at_3
value: 36.472
- type: recall_at_5
value: 44.175
- type: recall_at_10
value: 55.371
- type: recall_at_20
value: 65.506
- type: recall_at_100
value: 87.252
- type: recall_at_1000
value: 99.78399999999999
- type: main_score
value: 41.357
task:
type: Retrieval
- dataset:
config: hin-eng
name: MTEB XPQARetrieval (hin-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 62.829
- type: ndcg_at_3
value: 68.886
- type: ndcg_at_5
value: 71.812
- type: ndcg_at_10
value: 74.405
- type: ndcg_at_20
value: 75.702
- type: ndcg_at_100
value: 77.08500000000001
- type: ndcg_at_1000
value: 77.377
- type: recall_at_1
value: 53.568000000000005
- type: recall_at_3
value: 73.095
- type: recall_at_5
value: 80.211
- type: recall_at_10
value: 87.229
- type: recall_at_20
value: 91.625
- type: recall_at_100
value: 97.844
- type: recall_at_1000
value: 100
- type: main_score
value: 74.405
task:
type: Retrieval
- dataset:
config: ita-ita
name: MTEB XPQARetrieval (ita-ita)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 73.303
- type: ndcg_at_3
value: 74.51299999999999
- type: ndcg_at_5
value: 76.383
- type: ndcg_at_10
value: 78.968
- type: ndcg_at_20
value: 80.331
- type: ndcg_at_100
value: 81.65599999999999
- type: ndcg_at_1000
value: 82.075
- type: recall_at_1
value: 50.68899999999999
- type: recall_at_3
value: 72.763
- type: recall_at_5
value: 80.85
- type: recall_at_10
value: 87.071
- type: recall_at_20
value: 91.62599999999999
- type: recall_at_100
value: 97.333
- type: recall_at_1000
value: 100
- type: main_score
value: 78.968
task:
type: Retrieval
- dataset:
config: eng-ita
name: MTEB XPQARetrieval (eng-ita)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 42.232
- type: ndcg_at_3
value: 46.231
- type: ndcg_at_5
value: 48.197
- type: ndcg_at_10
value: 52.217
- type: ndcg_at_20
value: 55.472
- type: ndcg_at_100
value: 58.803000000000004
- type: ndcg_at_1000
value: 60.321000000000005
- type: recall_at_1
value: 26.368000000000002
- type: recall_at_3
value: 46.709
- type: recall_at_5
value: 54.721
- type: recall_at_10
value: 64.46
- type: recall_at_20
value: 74.997
- type: recall_at_100
value: 89.527
- type: recall_at_1000
value: 99.698
- type: main_score
value: 52.217
task:
type: Retrieval
- dataset:
config: ita-eng
name: MTEB XPQARetrieval (ita-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 68.326
- type: ndcg_at_3
value: 70.71499999999999
- type: ndcg_at_5
value: 72.748
- type: ndcg_at_10
value: 75.31
- type: ndcg_at_20
value: 76.958
- type: ndcg_at_100
value: 78.66300000000001
- type: ndcg_at_1000
value: 79.089
- type: recall_at_1
value: 46.583999999999996
- type: recall_at_3
value: 69.887
- type: recall_at_5
value: 78.10000000000001
- type: recall_at_10
value: 84.329
- type: recall_at_20
value: 89.51
- type: recall_at_100
value: 97.235
- type: recall_at_1000
value: 100
- type: main_score
value: 75.31
task:
type: Retrieval
- dataset:
config: jpn-jpn
name: MTEB XPQARetrieval (jpn-jpn)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 72
- type: ndcg_at_3
value: 74.005
- type: ndcg_at_5
value: 75.411
- type: ndcg_at_10
value: 77.12
- type: ndcg_at_20
value: 78.625
- type: ndcg_at_100
value: 80.281
- type: ndcg_at_1000
value: 80.682
- type: recall_at_1
value: 46.988
- type: recall_at_3
value: 72.36200000000001
- type: recall_at_5
value: 79.501
- type: recall_at_10
value: 83.83
- type: recall_at_20
value: 88.907
- type: recall_at_100
value: 96.739
- type: recall_at_1000
value: 99.636
- type: main_score
value: 77.12
task:
type: Retrieval
- dataset:
config: eng-jpn
name: MTEB XPQARetrieval (eng-jpn)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 43.758
- type: ndcg_at_3
value: 45.513999999999996
- type: ndcg_at_5
value: 47.93
- type: ndcg_at_10
value: 51.983999999999995
- type: ndcg_at_20
value: 54.544000000000004
- type: ndcg_at_100
value: 58.022
- type: ndcg_at_1000
value: 59.843
- type: recall_at_1
value: 25.543
- type: recall_at_3
value: 44.374
- type: recall_at_5
value: 53.86300000000001
- type: recall_at_10
value: 63.756
- type: recall_at_20
value: 72.14699999999999
- type: recall_at_100
value: 87.58200000000001
- type: recall_at_1000
value: 99.295
- type: main_score
value: 51.983999999999995
task:
type: Retrieval
- dataset:
config: jpn-eng
name: MTEB XPQARetrieval (jpn-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 68.978
- type: ndcg_at_3
value: 71.019
- type: ndcg_at_5
value: 72.697
- type: ndcg_at_10
value: 75.267
- type: ndcg_at_20
value: 76.655
- type: ndcg_at_100
value: 78.388
- type: ndcg_at_1000
value: 78.899
- type: recall_at_1
value: 44.958999999999996
- type: recall_at_3
value: 69.56400000000001
- type: recall_at_5
value: 77.082
- type: recall_at_10
value: 83.646
- type: recall_at_20
value: 88.238
- type: recall_at_100
value: 96.194
- type: recall_at_1000
value: 99.818
- type: main_score
value: 75.267
task:
type: Retrieval
- dataset:
config: kor-kor
name: MTEB XPQARetrieval (kor-kor)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 33.18
- type: ndcg_at_3
value: 35.311
- type: ndcg_at_5
value: 38.366
- type: ndcg_at_10
value: 41.654
- type: ndcg_at_20
value: 44.244
- type: ndcg_at_100
value: 49.001
- type: ndcg_at_1000
value: 51.01
- type: recall_at_1
value: 23.201
- type: recall_at_3
value: 37.011
- type: recall_at_5
value: 44.493
- type: recall_at_10
value: 53.489
- type: recall_at_20
value: 62.548
- type: recall_at_100
value: 85.55
- type: recall_at_1000
value: 100
- type: main_score
value: 41.654
task:
type: Retrieval
- dataset:
config: eng-kor
name: MTEB XPQARetrieval (eng-kor)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 34.404
- type: ndcg_at_3
value: 35.821
- type: ndcg_at_5
value: 37.268
- type: ndcg_at_10
value: 40.967
- type: ndcg_at_20
value: 43.509
- type: ndcg_at_100
value: 49.326
- type: ndcg_at_1000
value: 51.410999999999994
- type: recall_at_1
value: 20.363999999999997
- type: recall_at_3
value: 35.293
- type: recall_at_5
value: 41.251
- type: recall_at_10
value: 50.766999999999996
- type: recall_at_20
value: 59.274
- type: recall_at_100
value: 86.669
- type: recall_at_1000
value: 100
- type: main_score
value: 40.967
task:
type: Retrieval
- dataset:
config: kor-eng
name: MTEB XPQARetrieval (kor-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 33.062000000000005
- type: ndcg_at_3
value: 35.619
- type: ndcg_at_5
value: 37.684
- type: ndcg_at_10
value: 40.986
- type: ndcg_at_20
value: 43.736999999999995
- type: ndcg_at_100
value: 48.632999999999996
- type: ndcg_at_1000
value: 50.78
- type: recall_at_1
value: 23.18
- type: recall_at_3
value: 37.235
- type: recall_at_5
value: 42.448
- type: recall_at_10
value: 51.395
- type: recall_at_20
value: 61.01
- type: recall_at_100
value: 84.382
- type: recall_at_1000
value: 100
- type: main_score
value: 40.986
task:
type: Retrieval
- dataset:
config: pol-pol
name: MTEB XPQARetrieval (pol-pol)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 46.115
- type: ndcg_at_3
value: 45.966
- type: ndcg_at_5
value: 48.119
- type: ndcg_at_10
value: 51.53
- type: ndcg_at_20
value: 54.447
- type: ndcg_at_100
value: 58.939
- type: ndcg_at_1000
value: 60.428000000000004
- type: recall_at_1
value: 27.641
- type: recall_at_3
value: 45.021
- type: recall_at_5
value: 52.580000000000005
- type: recall_at_10
value: 61.141999999999996
- type: recall_at_20
value: 70.588
- type: recall_at_100
value: 90.29700000000001
- type: recall_at_1000
value: 99.851
- type: main_score
value: 51.53
task:
type: Retrieval
- dataset:
config: eng-pol
name: MTEB XPQARetrieval (eng-pol)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 32.357
- type: ndcg_at_3
value: 31.573
- type: ndcg_at_5
value: 33.046
- type: ndcg_at_10
value: 37.364999999999995
- type: ndcg_at_20
value: 40.407
- type: ndcg_at_100
value: 45.965
- type: ndcg_at_1000
value: 48.982
- type: recall_at_1
value: 14.865999999999998
- type: recall_at_3
value: 28.51
- type: recall_at_5
value: 35.827999999999996
- type: recall_at_10
value: 46.11
- type: recall_at_20
value: 55.498999999999995
- type: recall_at_100
value: 79.73
- type: recall_at_1000
value: 99.236
- type: main_score
value: 37.364999999999995
task:
type: Retrieval
- dataset:
config: pol-eng
name: MTEB XPQARetrieval (pol-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 43.114999999999995
- type: ndcg_at_3
value: 42.306
- type: ndcg_at_5
value: 44.484
- type: ndcg_at_10
value: 48.374
- type: ndcg_at_20
value: 51.347
- type: ndcg_at_100
value: 56.223
- type: ndcg_at_1000
value: 57.93899999999999
- type: recall_at_1
value: 25.746000000000002
- type: recall_at_3
value: 41.160000000000004
- type: recall_at_5
value: 48.256
- type: recall_at_10
value: 58.038999999999994
- type: recall_at_20
value: 67.499
- type: recall_at_100
value: 88.912
- type: recall_at_1000
value: 99.85000000000001
- type: main_score
value: 48.374
task:
type: Retrieval
- dataset:
config: por-por
name: MTEB XPQARetrieval (por-por)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 47.25
- type: ndcg_at_3
value: 46.225
- type: ndcg_at_5
value: 47.813
- type: ndcg_at_10
value: 51.383
- type: ndcg_at_20
value: 54.291
- type: ndcg_at_100
value: 58.434
- type: ndcg_at_1000
value: 60.07
- type: recall_at_1
value: 25.394
- type: recall_at_3
value: 43.446
- type: recall_at_5
value: 51.037
- type: recall_at_10
value: 59.61
- type: recall_at_20
value: 68.925
- type: recall_at_100
value: 88.277
- type: recall_at_1000
value: 99.44800000000001
- type: main_score
value: 51.383
task:
type: Retrieval
- dataset:
config: eng-por
name: MTEB XPQARetrieval (eng-por)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 29.5
- type: ndcg_at_3
value: 29.971999999999998
- type: ndcg_at_5
value: 31.513999999999996
- type: ndcg_at_10
value: 35.449999999999996
- type: ndcg_at_20
value: 38.912
- type: ndcg_at_100
value: 44.695
- type: ndcg_at_1000
value: 47.309
- type: recall_at_1
value: 14.335
- type: recall_at_3
value: 27.839999999999996
- type: recall_at_5
value: 34.737
- type: recall_at_10
value: 44.358
- type: recall_at_20
value: 55.65
- type: recall_at_100
value: 82.077
- type: recall_at_1000
value: 99.44800000000001
- type: main_score
value: 35.449999999999996
task:
type: Retrieval
- dataset:
config: por-eng
name: MTEB XPQARetrieval (por-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 46.048
- type: ndcg_at_3
value: 45.519
- type: ndcg_at_5
value: 47.693999999999996
- type: ndcg_at_10
value: 51.535
- type: ndcg_at_20
value: 54.179
- type: ndcg_at_100
value: 58.416999999999994
- type: ndcg_at_1000
value: 59.955000000000005
- type: recall_at_1
value: 25.325999999999997
- type: recall_at_3
value: 42.779
- type: recall_at_5
value: 51.453
- type: recall_at_10
value: 60.876
- type: recall_at_20
value: 69.184
- type: recall_at_100
value: 88.97699999999999
- type: recall_at_1000
value: 99.58200000000001
- type: main_score
value: 51.535
task:
type: Retrieval
- dataset:
config: tam-tam
name: MTEB XPQARetrieval (tam-tam)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 31.968999999999998
- type: ndcg_at_3
value: 34.555
- type: ndcg_at_5
value: 36.504999999999995
- type: ndcg_at_10
value: 38.958
- type: ndcg_at_20
value: 40.77
- type: ndcg_at_100
value: 43.779
- type: ndcg_at_1000
value: 47.388999999999996
- type: recall_at_1
value: 21.13
- type: recall_at_3
value: 35.838
- type: recall_at_5
value: 41.535
- type: recall_at_10
value: 48.075
- type: recall_at_20
value: 54.290000000000006
- type: recall_at_100
value: 68.325
- type: recall_at_1000
value: 95.62
- type: main_score
value: 38.958
task:
type: Retrieval
- dataset:
config: eng-tam
name: MTEB XPQARetrieval (eng-tam)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 12.531999999999998
- type: ndcg_at_3
value: 12.849
- type: ndcg_at_5
value: 13.979
- type: ndcg_at_10
value: 16.573
- type: ndcg_at_20
value: 18.861
- type: ndcg_at_100
value: 23.779
- type: ndcg_at_1000
value: 29.859
- type: recall_at_1
value: 7.388999999999999
- type: recall_at_3
value: 12.531999999999998
- type: recall_at_5
value: 16.279
- type: recall_at_10
value: 23.099
- type: recall_at_20
value: 30.697000000000003
- type: recall_at_100
value: 53.608
- type: recall_at_1000
value: 94.719
- type: main_score
value: 16.573
task:
type: Retrieval
- dataset:
config: tam-eng
name: MTEB XPQARetrieval (tam-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 21.066
- type: ndcg_at_3
value: 23.677999999999997
- type: ndcg_at_5
value: 25.851000000000003
- type: ndcg_at_10
value: 28.615000000000002
- type: ndcg_at_20
value: 30.817
- type: ndcg_at_100
value: 34.874
- type: ndcg_at_1000
value: 39.24
- type: recall_at_1
value: 15.037
- type: recall_at_3
value: 25.285999999999998
- type: recall_at_5
value: 30.717
- type: recall_at_10
value: 37.722
- type: recall_at_20
value: 44.927
- type: recall_at_100
value: 63.917
- type: recall_at_1000
value: 96.145
- type: main_score
value: 28.615000000000002
task:
type: Retrieval
- dataset:
config: cmn-cmn
name: MTEB XPQARetrieval (cmn-cmn)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 65.049
- type: ndcg_at_3
value: 65.534
- type: ndcg_at_5
value: 67.498
- type: ndcg_at_10
value: 70.812
- type: ndcg_at_20
value: 73.026
- type: ndcg_at_100
value: 75.316
- type: ndcg_at_1000
value: 75.882
- type: recall_at_1
value: 41.357
- type: recall_at_3
value: 63.176
- type: recall_at_5
value: 71.381
- type: recall_at_10
value: 79.47
- type: recall_at_20
value: 86.616
- type: recall_at_100
value: 96.36099999999999
- type: recall_at_1000
value: 100
- type: main_score
value: 70.812
task:
type: Retrieval
- dataset:
config: eng-cmn
name: MTEB XPQARetrieval (eng-cmn)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 35.073
- type: ndcg_at_3
value: 35.782000000000004
- type: ndcg_at_5
value: 36.99
- type: ndcg_at_10
value: 40.974
- type: ndcg_at_20
value: 43.971
- type: ndcg_at_100
value: 49.165
- type: ndcg_at_1000
value: 51.93
- type: recall_at_1
value: 20.057
- type: recall_at_3
value: 34.064
- type: recall_at_5
value: 40.831
- type: recall_at_10
value: 50.33
- type: recall_at_20
value: 59.306000000000004
- type: recall_at_100
value: 82.231
- type: recall_at_1000
value: 99.759
- type: main_score
value: 40.974
task:
type: Retrieval
- dataset:
config: cmn-eng
name: MTEB XPQARetrieval (cmn-eng)
revision: c99d599f0a6ab9b85b065da6f9d94f9cf731679f
split: test
type: jinaai/xpqa
metrics:
- type: ndcg_at_1
value: 57.68299999999999
- type: ndcg_at_3
value: 60.089000000000006
- type: ndcg_at_5
value: 62.217999999999996
- type: ndcg_at_10
value: 65.81700000000001
- type: ndcg_at_20
value: 67.886
- type: ndcg_at_100
value: 70.804
- type: ndcg_at_1000
value: 71.54
- type: recall_at_1
value: 36.146
- type: recall_at_3
value: 59.035000000000004
- type: recall_at_5
value: 67.376
- type: recall_at_10
value: 76.213
- type: recall_at_20
value: 82.756
- type: recall_at_100
value: 95.341
- type: recall_at_1000
value: 100
- type: main_score
value: 65.81700000000001
task:
type: Retrieval
tags:
- sentence-transformers
- transformers
- sentence-similarity
- mteb
INF-Retriever-v1
Model Overview
INF-Retriever-v1 is an LLM-based dense retrieval model developed by INF TECH. It is built upon the gte-Qwen2-7B-instruct model and specifically fine-tuned to excel in retrieval tasks, particularly for Chinese and English data.
As of January 23, 2025, INF-Retriever-v1 ranks both No.1 on the Automated Heterogeneous Information Retrieval Benchmark of version 24.04 & 24.05(AIR-Bench), showcasing its cutting-edge performance in heterogeneous information retrieval tasks.
Key Features
Optimized for Chinese and English retrieval: The model has been specifically fine-tuned with retrieval-focused datasets in both languages, significantly improving its accuracy and efficiency for a variety of retrieval scenarios.
Top-tier performance: INF-Retriever-v1 has achieved outstanding results on the AIR-Bench leaderboard, making it a top choice for heterogeneous information retrieval tasks across various domains.
Usage
Sentence Transformers
from sentence_transformers import SentenceTransformer
model = SentenceTransformer("infly/inf-retriever-v1", trust_remote_code=True)
# In case you want to reduce the maximum length:
model.max_seq_length = 8192
queries = [
"how much protein should a female eat",
"summit define",
]
documents = [
"As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.",
"Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments.",
]
query_embeddings = model.encode(queries, prompt_name="query")
document_embeddings = model.encode(documents)
scores = (query_embeddings @ document_embeddings.T) * 100
print(scores.tolist())
# [[86.8702392578125, 67.82364654541016], [59.51014709472656, 82.33668518066406]]
Transformers
import torch
import torch.nn.functional as F
from torch import Tensor
from transformers import AutoTokenizer, AutoModel
def last_token_pool(last_hidden_states: Tensor,
attention_mask: Tensor) -> Tensor:
left_padding = (attention_mask[:, -1].sum() == attention_mask.shape[0])
if left_padding:
return last_hidden_states[:, -1]
else:
sequence_lengths = attention_mask.sum(dim=1) - 1
batch_size = last_hidden_states.shape[0]
return last_hidden_states[torch.arange(batch_size, device=last_hidden_states.device), sequence_lengths]
def get_detailed_instruct(task_description: str, query: str) -> str:
return f'Instruct: {task_description}\nQuery: {query}'
# Each query must come with a one-sentence instruction that describes the task
task = 'Given a web search query, retrieve relevant passages that answer the query'
queries = [
get_detailed_instruct(task, 'how much protein should a female eat'),
get_detailed_instruct(task, 'summit define')
]
# No need to add instruction for retrieval documents
documents = [
"As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.",
"Definition of summit for English Language Learners. : 1 the highest point of a mountain : the top of a mountain. : 2 the highest level. : 3 a meeting or series of meetings between the leaders of two or more governments."
]
input_texts = queries + documents
tokenizer = AutoTokenizer.from_pretrained('infly/inf-retriever-v1', trust_remote_code=True)
model = AutoModel.from_pretrained('infly/inf-retriever-v1', trust_remote_code=True)
max_length = 8192
# Tokenize the input texts
batch_dict = tokenizer(input_texts, max_length=max_length, padding=True, truncation=True, return_tensors='pt')
outputs = model(**batch_dict)
embeddings = last_token_pool(outputs.last_hidden_state, batch_dict['attention_mask'])
# normalize embeddings
embeddings = F.normalize(embeddings, p=2, dim=1)
scores = (embeddings[:2] @ embeddings[2:].T) * 100
print(scores.tolist())
# [[86.87025451660156, 67.82366180419922], [59.510135650634766, 82.33667755126953]]
Evaluation
AIR-Bench
INF-Retriever-v1 has demonstrated superior retrieval capabilities across multiple domains and languages. The results from the Automated Heterogeneous Information Retrieval Benchmark (AIR-Bench) as of January 23, 2025, are as follows:
AIR-Bench_24.04 (Bilingual, EN & ZH)
| Model Name | Average⬆️ | wiki_en | wiki_zh | web_en | web_zh | healthcare_en | healthcare_zh | law_en | arxiv_en | news_en | news_zh | finance_en | finance_zh | msmarco_en |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| E5-mistral-7b-instruct | 45.26 | 61.67 | 55.97 | 44.41 | 45.96 | 56.32 | 35.79 | 19.32 | 44.78 | 48.18 | 35.99 | 54.79 | 26.11 | 59.03 |
| BGE-M3 | 46.65 | 60.49 | 62.36 | 47.35 | 50.38 | 49.1 | 42.38 | 26.68 | 40.76 | 48.04 | 40.75 | 51.52 | 32.18 | 54.4 |
| BGE-Multilingual-Gemma2 | 46.83 | 63.71 | 67.3 | 50.38 | 53.24 | 47.24 | 42.13 | 22.58 | 23.28 | 50.91 | 44.02 | 49.3 | 31.6 | 63.14 |
| GTE-Qwen2-7B-instruct | 48.38 | 63.46 | 66.44 | 51.2 | 51.98 | 54.2 | 38.82 | 22.31 | 40.27 | 54.07 | 43.03 | 58.2 | 26.63 | 58.39 |
| INF-Retriever-v1 | 52.56 | 65.25 | 68.44 | 52.13 | 56.6 | 56.96 | 42.03 | 34.51 | 50.62 | 53.32 | 50.02 | 58.34 | 35.42 | 59.64 |
AIR-Bench_24.05 (Multilingual, 13 languages)
Although INF-Retriever-v1 has been fine-tuned exclusively on English and Chinese, it continues to perform exceptionally well across other languages, securing the No. 1 position on this multilingual benchmark.
| Model Name | Average⬆️ | wiki_en | wiki_zh | wiki_ar | wiki_bn | wiki_de | wiki_es | wiki_fa | wiki_fr | wiki_hi | wiki_id | wiki_ja | wiki_ko | wiki_ru | web_en | web_zh | web_ar | web_bn | web_de | web_es | web_fa | web_fr | web_hi | web_id | web_ja | web_ko | web_ru | healthcare_en | healthcare_zh | healthcare_de | healthcare_es | healthcare_fr | law_en | law_de | law_fr | arxiv_en | science_ru | news_en | news_zh | news_ar | news_bn | news_de | news_es | news_fa | news_fr | news_hi | news_id | news_ja | news_ko | news_ru | finance_en | finance_zh | finance_ar | finance_fr |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| GTE-Qwen2-7B-instruct | 50.05 | 73.59 | 67.5 | 59.44 | 58.17 | 63.96 | 67.62 | 57.05 | 70.32 | 60.54 | 61.81 | 62.88 | 59.17 | 62.95 | 58.99 | 51.66 | 55.56 | 51.45 | 48.62 | 54.11 | 49.54 | 55.16 | 53.06 | 55.51 | 57.27 | 57.54 | 55.88 | 54.46 | 38.66 | 53.92 | 53.78 | 30.29 | 22.75 | 13.18 | 13.15 | 41.32 | 45.21 | 52.74 | 43.17 | 37.63 | 61.31 | 44.89 | 45.21 | 30.1 | 49.76 | 30.28 | 46.44 | 44.13 | 47.19 | 46.55 | 59.23 | 34.61 | 43.56 | 39.57 |
| Multilingual-E5-large-instruct | 51.11 | 68.62 | 62.82 | 63.21 | 64.45 | 65.81 | 68.1 | 64.2 | 69.72 | 71.81 | 66.36 | 64.12 | 64.79 | 62.57 | 41.58 | 47.06 | 56.4 | 56.17 | 50.87 | 52.24 | 58.68 | 50.2 | 56.32 | 54.49 | 54.89 | 55.81 | 54.97 | 54.02 | 39.76 | 52.06 | 51.74 | 36.64 | 16.9 | 15.59 | 15.12 | 39.52 | 56.86 | 44.28 | 35.46 | 48.2 | 49.31 | 47.84 | 45.99 | 45.59 | 50.58 | 39.66 | 48.59 | 47.6 | 50.52 | 48.81 | 52.79 | 37.72 | 48.95 | 42.74 |
| BGE-M3 | 51.31 | 69.7 | 63.52 | 59.65 | 64.33 | 64.68 | 65.4 | 61.14 | 66.04 | 69.02 | 66.3 | 60.86 | 62.36 | 60.18 | 53.88 | 50.2 | 52.53 | 55.53 | 51.89 | 51.78 | 55.81 | 51.46 | 57.06 | 53.14 | 54.75 | 55.28 | 54.53 | 49.05 | 42.31 | 49 | 53.05 | 39.29 | 26.95 | 20.11 | 20.2 | 41.64 | 55.18 | 47.34 | 41 | 44.93 | 59.03 | 47.87 | 44.7 | 43.81 | 49.52 | 42.12 | 47.45 | 47.09 | 48.14 | 48.31 | 52.92 | 40.23 | 45.76 | 41.44 |
| BGE-Multilingual-Gemma2 | 54.46 | 72.8 | 68.64 | 63.42 | 69.48 | 67.91 | 71.79 | 67.57 | 71.28 | 75.39 | 68.91 | 68.29 | 66.78 | 64.15 | 56.48 | 53.04 | 59.97 | 59.68 | 57.72 | 58.2 | 62.43 | 59.54 | 64.5 | 60 | 60.26 | 59.64 | 60.12 | 47.48 | 42.35 | 55.4 | 63.13 | 45.13 | 22.6 | 15.75 | 14.29 | 24 | 44.13 | 50.29 | 43.42 | 48.41 | 58.77 | 52.05 | 49.9 | 43.4 | 56.8 | 44.89 | 50.65 | 51.51 | 51.64 | 51.48 | 50.08 | 39.23 | 50.25 | 51.1 |
| INF-Retriever-v1 | 54.47 | 73.52 | 69.45 | 63.13 | 61.58 | 66.8 | 69.29 | 63.03 | 69.74 | 69.02 | 68.63 | 63.45 | 64.44 | 62.74 | 57.6 | 56.46 | 58.48 | 53.7 | 55.2 | 57.08 | 53.27 | 57.35 | 55.64 | 58.85 | 59.52 | 60.01 | 58.79 | 57.03 | 41.82 | 55.46 | 57.6 | 43.25 | 34.76 | 21.75 | 21.87 | 51.38 | 59.72 | 52.7 | 49.78 | 49.11 | 43.62 | 51.47 | 49.52 | 40.43 | 54.54 | 38.57 | 51.06 | 51.12 | 53.15 | 51.88 | 59.44 | 44.13 | 50.71 | 44.2 |
Contributors
Supervisors
Wei Chu • Yinghui Xu • Yuan Qi
INF memory team
Junhan Yang ([email protected]) • Jiahe Wan • Yichen Yao ([email protected])