Spaces:
Runtime error
Runtime error
Commit
Β·
d6696d6
1
Parent(s):
7545429
feat: deprecate old leaderboards
Browse files
app.py
CHANGED
|
@@ -59,8 +59,56 @@ def main():
|
|
| 59 |
|
| 60 |
with gr.Blocks(css=css) as block:
|
| 61 |
with gr.Tabs():
|
| 62 |
-
with gr.TabItem("
|
| 63 |
-
gr.Markdown("#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models π")
|
| 65 |
|
| 66 |
gr.Markdown(
|
|
@@ -144,8 +192,15 @@ def main():
|
|
| 144 |
```
|
| 145 |
"""
|
| 146 |
)
|
| 147 |
-
with gr.TabItem("
|
| 148 |
-
gr.Markdown(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 149 |
gr.Markdown("### A harder dataset benchmark for visual document retrieval π")
|
| 150 |
|
| 151 |
gr.Markdown(
|
|
@@ -247,48 +302,6 @@ def main():
|
|
| 247 |
```
|
| 248 |
"""
|
| 249 |
)
|
| 250 |
-
|
| 251 |
-
with gr.TabItem("π Submit your model"):
|
| 252 |
-
gr.Markdown("# How to Submit a New Model to the Leaderboard")
|
| 253 |
-
gr.Markdown(
|
| 254 |
-
"""
|
| 255 |
-
To submit a new model to the ViDoRe leaderboard, follow these steps:
|
| 256 |
-
|
| 257 |
-
1. **Evaluate your model**:
|
| 258 |
-
- Follow the evaluation script provided in the [ViDoRe GitHub repository](https://github.com/illuin-tech/vidore-benchmark/)
|
| 259 |
-
|
| 260 |
-
2. **Format your submission file**:
|
| 261 |
-
- The submission file should automatically be generated, and named `results.json` with the
|
| 262 |
-
following structure:
|
| 263 |
-
```json
|
| 264 |
-
{
|
| 265 |
-
"dataset_name_1": {
|
| 266 |
-
"metric_1": score_1,
|
| 267 |
-
"metric_2": score_2,
|
| 268 |
-
...
|
| 269 |
-
},
|
| 270 |
-
"dataset_name_2": {
|
| 271 |
-
"metric_1": score_1,
|
| 272 |
-
"metric_2": score_2,
|
| 273 |
-
...
|
| 274 |
-
},
|
| 275 |
-
}
|
| 276 |
-
```
|
| 277 |
-
- The dataset names should be the same as the ViDoRe and ViDoRe 2 dataset names listed in the following
|
| 278 |
-
collections: [ViDoRe Benchmark](https://huggingface.co/collections/vidore/vidore-benchmark-667173f98e70a1c0fa4db00d) and [ViDoRe Benchmark 2](vidore/vidore-benchmark-v2-dev-67ae03e3924e85b36e7f53b0).
|
| 279 |
-
|
| 280 |
-
3. **Submit your model**:
|
| 281 |
-
- Create a public HuggingFace model repository with your model.
|
| 282 |
-
- Add the tag `vidore` to your model in the metadata of the model card and place the
|
| 283 |
-
`results.json` file at the root.
|
| 284 |
-
|
| 285 |
-
And you're done! Your model will appear on the leaderboard when you click refresh! Once the space
|
| 286 |
-
gets rebooted, it will appear on startup.
|
| 287 |
-
|
| 288 |
-
Note: For proper hyperlink redirection, please ensure that your model repository name is in
|
| 289 |
-
kebab-case, e.g. `my-model-name`.
|
| 290 |
-
"""
|
| 291 |
-
)
|
| 292 |
|
| 293 |
block.queue(max_size=10).launch(debug=True)
|
| 294 |
|
|
|
|
| 59 |
|
| 60 |
with gr.Blocks(css=css) as block:
|
| 61 |
with gr.Tabs():
|
| 62 |
+
with gr.TabItem("π Submit your model"):
|
| 63 |
+
gr.Markdown("# How to Submit a New Model to the Leaderboard")
|
| 64 |
+
gr.Markdown(
|
| 65 |
+
"""
|
| 66 |
+
To submit a new model to the ViDoRe leaderboard, follow these steps:
|
| 67 |
+
|
| 68 |
+
1. **Evaluate your model**:
|
| 69 |
+
- Follow the evaluation script provided in the [ViDoRe GitHub repository](https://github.com/illuin-tech/vidore-benchmark/)
|
| 70 |
+
|
| 71 |
+
2. **Format your submission file**:
|
| 72 |
+
- The submission file should automatically be generated, and named `results.json` with the
|
| 73 |
+
following structure:
|
| 74 |
+
```json
|
| 75 |
+
{
|
| 76 |
+
"dataset_name_1": {
|
| 77 |
+
"metric_1": score_1,
|
| 78 |
+
"metric_2": score_2,
|
| 79 |
+
...
|
| 80 |
+
},
|
| 81 |
+
"dataset_name_2": {
|
| 82 |
+
"metric_1": score_1,
|
| 83 |
+
"metric_2": score_2,
|
| 84 |
+
...
|
| 85 |
+
},
|
| 86 |
+
}
|
| 87 |
+
```
|
| 88 |
+
- The dataset names should be the same as the ViDoRe and ViDoRe 2 dataset names listed in the following
|
| 89 |
+
collections: [ViDoRe Benchmark](https://huggingface.co/collections/vidore/vidore-benchmark-667173f98e70a1c0fa4db00d) and [ViDoRe Benchmark 2](vidore/vidore-benchmark-v2-dev-67ae03e3924e85b36e7f53b0).
|
| 90 |
+
|
| 91 |
+
3. **Submit your model**:
|
| 92 |
+
- Create a public HuggingFace model repository with your model.
|
| 93 |
+
- Add the tag `vidore` to your model in the metadata of the model card and place the
|
| 94 |
+
`results.json` file at the root.
|
| 95 |
+
|
| 96 |
+
And you're done! Your model will appear on the leaderboard when you click refresh! Once the space
|
| 97 |
+
gets rebooted, it will appear on startup.
|
| 98 |
+
|
| 99 |
+
Note: For proper hyperlink redirection, please ensure that your model repository name is in
|
| 100 |
+
kebab-case, e.g. `my-model-name`.
|
| 101 |
+
"""
|
| 102 |
+
)
|
| 103 |
+
with gr.TabItem("[Deprecated] ViDoRe V1"):
|
| 104 |
+
gr.Markdown(
|
| 105 |
+
"## <span style='color:red'>Deprecation notice: This leaderboard contains the results computed with the "
|
| 106 |
+
"[vidore-benchmark](https://github.com/illuin-tech/vidore-benchmark) package, "
|
| 107 |
+
"which is no longer maintained. Results should be computed using the "
|
| 108 |
+
"[mteb](https://github.com/embeddings-benchmark/mteb) package as described "
|
| 109 |
+
"[here](https://github.com/illuin-tech/vidore-benchmark/blob/main/README.md).</span>"
|
| 110 |
+
)
|
| 111 |
+
gr.Markdown("# [Deprecated] ViDoRe: The Visual Document Retrieval Benchmark 1 ππ")
|
| 112 |
gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models π")
|
| 113 |
|
| 114 |
gr.Markdown(
|
|
|
|
| 192 |
```
|
| 193 |
"""
|
| 194 |
)
|
| 195 |
+
with gr.TabItem("[Deprecated] ViDoRe V2"):
|
| 196 |
+
gr.Markdown(
|
| 197 |
+
"## <span style='color:red'>Deprecation notice: This leaderboard contains the results computed with the "
|
| 198 |
+
"[vidore-benchmark](https://github.com/illuin-tech/vidore-benchmark) package, "
|
| 199 |
+
"which is no longer maintained. Results should be computed using the "
|
| 200 |
+
"[mteb](https://github.com/embeddings-benchmark/mteb) package as described "
|
| 201 |
+
"[here](https://github.com/illuin-tech/vidore-benchmark/blob/main/README.md).</span>"
|
| 202 |
+
)
|
| 203 |
+
gr.Markdown("# [Deprecated] ViDoRe V2: A new visual Document Retrieval Benchmark ππ")
|
| 204 |
gr.Markdown("### A harder dataset benchmark for visual document retrieval π")
|
| 205 |
|
| 206 |
gr.Markdown(
|
|
|
|
| 302 |
```
|
| 303 |
"""
|
| 304 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 305 |
|
| 306 |
block.queue(max_size=10).launch(debug=True)
|
| 307 |
|