Use MTEB results in new leaderboard (#8)

- feat: deprecate old leaderboards (d6696d6acb32dff8dacc826328c0952a01353cfb)
- feat: add new leaderboard based on mteb results (a212b4a0cd1cd9c7b27383aa5f0366421be7645c)
- feat: add model size column (b5279c6709dde10405810e48634d5b1601c74cef)
- docs: update documentation on how to add your model (1c161e2337749f98d0876f4b4149c018fcf129a9)


Co-authored-by: Antonio Loison <[email protected]>

app.py

@@ -1,6 +1,7 @@
 import gradio as gr
 
 from app.utils import add_rank_and_format, filter_models, get_refresh_function
+from data.deprecated_model_handler import DeprecatedModelHandler
 from data.model_handler import ModelHandler
 
 METRICS = [
@@ -16,23 +17,43 @@ METRICS = [
 
 
 def main():
+    # Get new results
     model_handler = ModelHandler()
     initial_metric = "ndcg_at_5"
 
     model_handler.get_vidore_data(initial_metric)
-    data_benchmark_1 = model_handler.compute_averages(initial_metric, benchmark_version=1)
+    data_benchmark_1 = model_handler.render_df(initial_metric, benchmark_version=1)
     data_benchmark_1 = add_rank_and_format(data_benchmark_1, benchmark_version=1)
 
-    data_benchmark_2 = model_handler.compute_averages(initial_metric, benchmark_version=2)
+    data_benchmark_2 = model_handler.render_df(initial_metric, benchmark_version=2)
     data_benchmark_2 = add_rank_and_format(data_benchmark_2, benchmark_version=2)
 
-    NUM_DATASETS_1 = len(data_benchmark_1.columns) - 3
-    NUM_SCORES_1 = len(data_benchmark_1) * NUM_DATASETS_1
-    NUM_MODELS_1 = len(data_benchmark_1)
+    num_datasets_1 = len(data_benchmark_1.columns) - 3
+    num_scores_1 = len(data_benchmark_1) * num_datasets_1
+    num_models_1 = len(data_benchmark_1)
 
-    NUM_DATASETS_2 = len(data_benchmark_2.columns) - 3
-    NUM_SCORES_2 = len(data_benchmark_2) * NUM_DATASETS_2
-    NUM_MODELS_2 = len(data_benchmark_2)
+    num_datasets_2 = len(data_benchmark_2.columns) - 3
+    num_scores_2 = len(data_benchmark_2) * num_datasets_2
+    num_models_2 = len(data_benchmark_2)
+
+    # Get deprecated results
+    deprecated_model_handler = DeprecatedModelHandler()
+    initial_metric = "ndcg_at_5"
+
+    deprecated_model_handler.get_vidore_data(initial_metric)
+    deprecated_data_benchmark_1 = deprecated_model_handler.render_df(initial_metric, benchmark_version=1)
+    deprecated_data_benchmark_1 = add_rank_and_format(deprecated_data_benchmark_1, benchmark_version=1)
+
+    deprecated_data_benchmark_2 = deprecated_model_handler.render_df(initial_metric, benchmark_version=2)
+    deprecated_data_benchmark_2 = add_rank_and_format(deprecated_data_benchmark_2, benchmark_version=2)
+
+    deprecated_num_datasets_1 = len(deprecated_data_benchmark_1.columns) - 3
+    deprecated_num_scores_1 = len(deprecated_data_benchmark_1) * deprecated_num_datasets_1
+    deprecated_num_models_1 = len(deprecated_data_benchmark_1)
+
+    deprecated_num_datasets_2 = len(deprecated_data_benchmark_2.columns) - 3
+    deprecated_num_scores_2 = len(deprecated_data_benchmark_2) * deprecated_num_datasets_2
+    deprecated_num_models_2 = len(deprecated_data_benchmark_2)
 
     css = """
     table > thead {
@@ -59,7 +80,7 @@ def main():
 
     with gr.Blocks(css=css) as block:
         with gr.Tabs():
-            with gr.TabItem("🏆 ViDoRe V1"):
+            with gr.TabItem("ViDoRe V1"):
                 gr.Markdown("# ViDoRe: The Visual Document Retrieval Benchmark 1 📚🔍")
                 gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models 👀")
 
@@ -70,7 +91,7 @@ def main():
                 Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics, tasks and models.
                 """
                 )
-                datasets_columns_1 = list(data_benchmark_1.columns[3:])
+                datasets_columns_1 = list(data_benchmark_1.columns[4:])
 
                 with gr.Row():
                     metric_dropdown_1 = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
@@ -83,17 +104,16 @@ def main():
                     )
 
                 with gr.Row():
-                    datatype_1 = ["number", "markdown"] + ["number"] * (NUM_DATASETS_1 + 1)
+                    datatype_1 = ["number", "markdown"] + ["number"] * (num_datasets_1 + 1)
                     dataframe_1 = gr.Dataframe(data_benchmark_1, datatype=datatype_1, type="pandas")
 
                 def update_data_1(metric, search_term, selected_columns):
                     model_handler.get_vidore_data(metric)
-                    data = model_handler.compute_averages(metric, benchmark_version=1)
+                    data = deprecated_model_handler.render_df(metric, benchmark_version=1)
                     data = add_rank_and_format(data, benchmark_version=1)
                     data = filter_models(data, search_term)
-                    # data = remove_duplicates(data)  # Add this line
                     if selected_columns:
-                        data = data[["Rank", "Model", "Average"] + selected_columns]
+                        data = data[["Rank", "Model", "Model Size (Million Parameters)", "Average"] + selected_columns]
                     return data
 
                 with gr.Row():
@@ -124,9 +144,9 @@ def main():
 
                 gr.Markdown(
                     f"""
-                - **Total Datasets**: {NUM_DATASETS_1}
-                - **Total Scores**: {NUM_SCORES_1}
-                - **Total Models**: {NUM_MODELS_1}
+                - **Total Datasets**: {num_datasets_1}
+                - **Total Scores**: {num_scores_1}
+                - **Total Models**: {num_models_1}
                 """
                     + r"""
                 Please consider citing:
@@ -144,7 +164,7 @@ def main():
                 ```
                 """
                 )
-            with gr.TabItem("🏆 ViDoRe V2"):
+            with gr.TabItem("ViDoRe V2"):
                 gr.Markdown("# ViDoRe V2: A new visual Document Retrieval Benchmark 📚🔍")
                 gr.Markdown("### A harder dataset benchmark for visual document retrieval 👀")
 
@@ -155,7 +175,7 @@ def main():
                 Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics and models.
                 """
                 )
-                datasets_columns_2 = list(data_benchmark_2.columns[3:])
+                datasets_columns_2 = list(data_benchmark_2.columns[4:])
 
                 with gr.Row():
                     metric_dropdown_2 = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
@@ -168,17 +188,17 @@ def main():
                     )
 
                 with gr.Row():
-                    datatype_2 = ["number", "markdown"] + ["number"] * (NUM_DATASETS_2 + 1)
+                    datatype_2 = ["number", "markdown"] + ["number"] * (num_datasets_2 + 1)
                     dataframe_2 = gr.Dataframe(data_benchmark_2, datatype=datatype_2, type="pandas")
 
                 def update_data_2(metric, search_term, selected_columns):
                     model_handler.get_vidore_data(metric)
-                    data = model_handler.compute_averages(metric, benchmark_version=2)
+                    data = deprecated_model_handler.render_df(metric, benchmark_version=2)
                     data = add_rank_and_format(data, benchmark_version=2)
                     data = filter_models(data, search_term)
                     # data = remove_duplicates(data)  # Add this line
                     if selected_columns:
-                        data = data[["Rank", "Model", "Average"] + selected_columns]
+                        data = data[["Rank", "Model", "Model Size (Million Parameters)", "Average"] + selected_columns]
                     return data
 
                 with gr.Row():
@@ -217,9 +237,9 @@ def main():
 
                 gr.Markdown(
                     f"""
-                - **Total Datasets**: {NUM_DATASETS_2}
-                - **Total Scores**: {NUM_SCORES_2}
-                - **Total Models**: {NUM_MODELS_2}
+                - **Total Datasets**: {num_datasets_2}
+                - **Total Scores**: {num_scores_2}
+                - **Total Models**: {num_models_2}
                 """
                     + r"""
                 Please consider citing:
@@ -247,7 +267,6 @@ def main():
                 ```
                 """
                 )
-            
             with gr.TabItem("📚 Submit your model"):
                 gr.Markdown("# How to Submit a New Model to the Leaderboard")
                 gr.Markdown(
@@ -255,32 +274,10 @@ def main():
                     To submit a new model to the ViDoRe leaderboard, follow these steps:
 
                     1. **Evaluate your model**:
-                       - Follow the evaluation script provided in the [ViDoRe GitHub repository](https://github.com/illuin-tech/vidore-benchmark/)
+                       - Follow the evaluation procedure provided in the [ViDoRe GitHub repository](https://github.com/illuin-tech/vidore-benchmark/) that uses MTEB.
 
                     2. **Format your submission file**:
-                        - The submission file should automatically be generated, and named `results.json` with the
-                        following structure:
-                        ```json
-                        {
-                            "dataset_name_1": {
-                                "metric_1": score_1,
-                                "metric_2": score_2,
-                                ...
-                            },
-                            "dataset_name_2": {
-                                "metric_1": score_1,
-                                "metric_2": score_2,
-                                ...
-                            },
-                        }
-                        ```
-                        - The dataset names should be the same as the ViDoRe and ViDoRe 2 dataset names listed in the following
-                        collections: [ViDoRe Benchmark](https://huggingface.co/collections/vidore/vidore-benchmark-667173f98e70a1c0fa4db00d) and [ViDoRe Benchmark 2](vidore/vidore-benchmark-v2-dev-67ae03e3924e85b36e7f53b0).
-
-                    3. **Submit your model**:
-                        - Create a public HuggingFace model repository with your model.
-                        - Add the tag `vidore` to your model in the metadata of the model card and place the
-                        `results.json` file at the root.
+                        - Add the generated files to [MTEB results](https://github.com/embeddings-benchmark/results) project. Check the [Colpali results](https://github.com/embeddings-benchmark/results/tree/main/results/vidore__colpali-v1.3/1b5c8929330df1a66de441a9b5409a878f0de5b0) for an example.
 
                     And you're done! Your model will appear on the leaderboard when you click refresh! Once the space
                     gets rebooted, it will appear on startup.
@@ -289,6 +286,210 @@ def main():
                     kebab-case, e.g. `my-model-name`.
                     """
                 )
+            with gr.TabItem("[Deprecated] ViDoRe V1"):
+                gr.Markdown(
+                    "## <span style='color:red'>Deprecation notice: This leaderboard contains the results computed with the "
+                    "[vidore-benchmark](https://github.com/illuin-tech/vidore-benchmark) package, "
+                    "which is no longer maintained. Results should be computed using the "
+                    "[mteb](https://github.com/embeddings-benchmark/mteb) package as described "
+                    "[here](https://github.com/illuin-tech/vidore-benchmark/blob/main/README.md).</span>"
+                )
+                gr.Markdown("## <span style='color:red'>Missing results in the new leaderboard are being added as they are re-computed.</span>")
+                gr.Markdown("# <span style='color:red'>[Deprecated]</span> ViDoRe: The Visual Document Retrieval Benchmark 1 📚🔍")
+                gr.Markdown("### From the paper - ColPali: Efficient Document Retrieval with Vision Language Models 👀")
+
+                gr.Markdown(
+                    """
+                Visual Document Retrieval Benchmark 1 leaderboard. To submit results, refer to the corresponding tab.  
+
+                Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics, tasks and models.
+                """
+                )
+                datasets_columns_1 = list(deprecated_data_benchmark_1.columns[3:])
+
+                with gr.Row():
+                    metric_dropdown_1 = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
+                    research_textbox_1 = gr.Textbox(
+                        placeholder="🔍 Search Models... [press enter]",
+                        label="Filter Models by Name",
+                    )
+                    column_checkboxes_1 = gr.CheckboxGroup(
+                        choices=datasets_columns_1, value=datasets_columns_1, label="Select Columns to Display"
+                    )
+
+                with gr.Row():
+                    datatype_1 = ["number", "markdown"] + ["number"] * (deprecated_num_datasets_1 + 1)
+                    dataframe_1 = gr.Dataframe(deprecated_data_benchmark_1, datatype=datatype_1, type="pandas")
+
+                def update_data_1(metric, search_term, selected_columns):
+                    deprecated_model_handler.get_vidore_data(metric)
+                    data = deprecated_model_handler.render_df(metric, benchmark_version=1)
+                    data = add_rank_and_format(data, benchmark_version=1)
+                    data = filter_models(data, search_term)
+                    # data = remove_duplicates(data)  # Add this line
+                    if selected_columns:
+                        data = data[["Rank", "Model", "Model Size (Million Parameters)", "Average"] + selected_columns]
+                    return data
+
+                with gr.Row():
+                    refresh_button_1 = gr.Button("Refresh")
+                    refresh_button_1.click(
+                        get_refresh_function(deprecated_model_handler, benchmark_version=1),
+                        inputs=[metric_dropdown_1],
+                        outputs=dataframe_1,
+                        concurrency_limit=20,
+                    )
+
+                # Automatically refresh the dataframe when the dropdown value changes
+                metric_dropdown_1.change(
+                    get_refresh_function(deprecated_model_handler, benchmark_version=1),
+                    inputs=[metric_dropdown_1],
+                    outputs=dataframe_1,
+                )
+                research_textbox_1.submit(
+                    lambda metric, search_term, selected_columns: update_data_1(metric, search_term, selected_columns),
+                    inputs=[metric_dropdown_1, research_textbox_1, column_checkboxes_1],
+                    outputs=dataframe_1,
+                )
+                column_checkboxes_1.change(
+                    lambda metric, search_term, selected_columns: update_data_1(metric, search_term, selected_columns),
+                    inputs=[metric_dropdown_1, research_textbox_1, column_checkboxes_1],
+                    outputs=dataframe_1,
+                )
+
+                gr.Markdown(
+                    f"""
+                - **Total Datasets**: {deprecated_num_datasets_1}
+                - **Total Scores**: {deprecated_num_scores_1}
+                - **Total Models**: {deprecated_num_models_1}
+                """
+                    + r"""
+                Please consider citing:
+
+                ```bibtex
+                @misc{faysse2024colpaliefficientdocumentretrieval,
+                  title={ColPali: Efficient Document Retrieval with Vision Language Models}, 
+                  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
+                  year={2024},
+                  eprint={2407.01449},
+                  archivePrefix={arXiv},
+                  primaryClass={cs.IR},
+                  url={https://arxiv.org/abs/2407.01449}, 
+                }
+                ```
+                """
+                )
+            with gr.TabItem("[Deprecated] ViDoRe V2"):
+                gr.Markdown(
+                    "## <span style='color:red'>Deprecation notice: This leaderboard contains the results computed with the "
+                    "[vidore-benchmark](https://github.com/illuin-tech/vidore-benchmark) package, "
+                    "which is no longer maintained. Results should be computed using the "
+                    "[mteb](https://github.com/embeddings-benchmark/mteb) package as described "
+                    "[here](https://github.com/illuin-tech/vidore-benchmark/blob/main/README.md).</span>"
+                )
+                gr.Markdown("## <span style='color:red'>Missing results in the new leaderboard are being added as they are re-computed.</span>")
+                gr.Markdown("# <span style='color:red'>[Deprecated]</span> ViDoRe V2: A new visual Document Retrieval Benchmark 📚🔍")
+                gr.Markdown("### A harder dataset benchmark for visual document retrieval 👀")
+
+                gr.Markdown(
+                    """
+                Visual Document Retrieval Benchmark 2 leaderboard. To submit results, refer to the corresponding tab.  
+
+                Refer to the [ColPali paper](https://arxiv.org/abs/2407.01449) for details on metrics and models.
+                """
+                )
+                datasets_columns_2 = list(deprecated_data_benchmark_2.columns[3:])
+
+                with gr.Row():
+                    metric_dropdown_2 = gr.Dropdown(choices=METRICS, value=initial_metric, label="Select Metric")
+                    research_textbox_2 = gr.Textbox(
+                        placeholder="🔍 Search Models... [press enter]",
+                        label="Filter Models by Name",
+                    )
+                    column_checkboxes_2 = gr.CheckboxGroup(
+                        choices=datasets_columns_2, value=datasets_columns_2, label="Select Columns to Display"
+                    )
+
+                with gr.Row():
+                    datatype_2 = ["number", "markdown"] + ["number"] * (deprecated_num_datasets_2 + 1)
+                    dataframe_2 = gr.Dataframe(deprecated_data_benchmark_2, datatype=datatype_2, type="pandas")
+
+                def update_data_2(metric, search_term, selected_columns):
+                    deprecated_model_handler.get_vidore_data(metric)
+                    data = deprecated_model_handler.render_df(metric, benchmark_version=2)
+                    data = add_rank_and_format(data, benchmark_version=2)
+                    data = filter_models(data, search_term)
+                    # data = remove_duplicates(data)  # Add this line
+                    if selected_columns:
+                        data = data[["Rank", "Model", "Model Size (Million Parameters)", "Average"] + selected_columns]
+                    return data
+
+                with gr.Row():
+                    refresh_button_2 = gr.Button("Refresh")
+                    refresh_button_2.click(
+                        get_refresh_function(deprecated_model_handler, benchmark_version=2),
+                        inputs=[metric_dropdown_2],
+                        outputs=dataframe_2,
+                        concurrency_limit=20,
+                    )
+
+                with gr.Row():
+                    gr.Markdown(
+                        """
+                    **Note**: For now, all models were evaluated using the vidore-benchmark package and custom retrievers on our side. 
+                    Those numbers are not numbers obtained from the organisations that released those models.
+                    """
+                    )
+
+                # Automatically refresh the dataframe when the dropdown value changes
+                metric_dropdown_2.change(
+                    get_refresh_function(deprecated_model_handler, benchmark_version=2),
+                    inputs=[metric_dropdown_2],
+                    outputs=dataframe_2,
+                )
+                research_textbox_2.submit(
+                    lambda metric, search_term, selected_columns: update_data_2(metric, search_term, selected_columns),
+                    inputs=[metric_dropdown_2, research_textbox_2, column_checkboxes_2],
+                    outputs=dataframe_2,
+                )
+                column_checkboxes_2.change(
+                    lambda metric, search_term, selected_columns: update_data_2(metric, search_term, selected_columns),
+                    inputs=[metric_dropdown_2, research_textbox_2, column_checkboxes_2],
+                    outputs=dataframe_2,
+                )
+
+                gr.Markdown(
+                    f"""
+                - **Total Datasets**: {deprecated_num_datasets_2}
+                - **Total Scores**: {deprecated_num_scores_2}
+                - **Total Models**: {deprecated_num_models_2}
+                """
+                    + r"""
+                Please consider citing:
+
+                ```bibtex
+                @misc{faysse2024colpaliefficientdocumentretrieval,
+                  title={ColPali: Efficient Document Retrieval with Vision Language Models}, 
+                  author={Manuel Faysse and Hugues Sibille and Tony Wu and Bilel Omrani and Gautier Viaud and Céline Hudelot and Pierre Colombo},
+                  year={2024},
+                  eprint={2407.01449},
+                  archivePrefix={arXiv},
+                  primaryClass={cs.IR},
+                  url={https://arxiv.org/abs/2407.01449}, 
+                }
+
+                @misc{macé2025vidorebenchmarkv2raising,
+                      title={ViDoRe Benchmark V2: Raising the Bar for Visual Retrieval}, 
+                      author={Quentin Macé and António Loison and Manuel Faysse},
+                      year={2025},
+                      eprint={2505.17166},
+                      archivePrefix={arXiv},
+                      primaryClass={cs.IR},
+                      url={https://arxiv.org/abs/2505.17166}, 
+                }
+                ```
+                """
+                )
 
     block.queue(max_size=10).launch(debug=True)
 

app/utils.py

@@ -1,9 +1,10 @@
-from data.model_handler import ModelHandler
+from data.deprecated_model_handler import DeprecatedModelHandler
 
 
 def make_clickable_model(model_name, link=None):
     if link is None:
-        desanitized_model_name = model_name.replace("_", "/")
+        desanitized_model_name = model_name.replace("__", "/")
+        desanitized_model_name = desanitized_model_name.replace("_", "/")
         desanitized_model_name = desanitized_model_name.replace("-thisisapoint-", ".")
 
         if "/captioning" in desanitized_model_name:
@@ -16,10 +17,38 @@ def make_clickable_model(model_name, link=None):
     return f'<a target="_blank" style="text-decoration: underline" href="{link}">{desanitized_model_name}</a>'
 
 
+def add_rank(df, benchmark_version=1):
+        df.fillna(0.0, inplace=True)
+        cols_to_rank = [
+            col
+            for col in df.columns
+            if col
+            not in [
+                "Model",
+                "Model Size (Million Parameters)",
+                "Memory Usage (GB, fp32)",
+                "Embedding Dimensions",
+                "Max Tokens",
+            ]
+        ]
+
+        if len(cols_to_rank) == 1:
+            df.sort_values(cols_to_rank[0], ascending=False, inplace=True)
+        else:
+            df.insert(len(df.columns) - len(cols_to_rank), "Average", df[cols_to_rank].mean(axis=1, skipna=False))
+            df.sort_values("Average", ascending=False, inplace=True)
+        df.insert(0, "Rank", list(range(1, len(df) + 1)))
+        # multiply values by 100 if they are floats and round to 1 decimal place
+        for col in df.columns:
+            if df[col].dtype == "float64" and col != "Model Size (Million Parameters)":
+                df[col] = df[col].apply(lambda x: round(x * 100, 1))
+        return df
+
+
 def add_rank_and_format(df, benchmark_version=1):
     df = df.reset_index()
     df = df.rename(columns={"index": "Model"})
-    df = ModelHandler.add_rank(df, benchmark_version)
+    df = add_rank(df, benchmark_version)
     df["Model"] = df["Model"].apply(make_clickable_model)
     # df = remove_duplicates(df)
     return df
@@ -36,7 +65,7 @@ def remove_duplicates(df):
 def get_refresh_function(model_handler, benchmark_version):
     def _refresh(metric):
         model_handler.get_vidore_data(metric)
-        data_task_category = model_handler.compute_averages(metric, benchmark_version)
+        data_task_category = model_handler.render_df(metric, benchmark_version)
         df = add_rank_and_format(data_task_category, benchmark_version)
         return df
 

data/dataset_handler.py

@@ -1,4 +1,22 @@
-VIDORE_DATASETS_KEYWORDS = [
+VIDORE_V1_MTEB_NAMES = [
+    "VidoreArxivQARetrieval",
+    "VidoreDocVQARetrieval",
+    "VidoreInfoVQARetrieval",
+    "VidoreShiftProjectRetrieval",
+    "VidoreSyntheticDocQAAIRetrieval",
+    "VidoreSyntheticDocQAEnergyRetrieval",
+    "VidoreSyntheticDocQAGovernmentReportsRetrieval",
+    "VidoreSyntheticDocQAHealthcareIndustryRetrieval",
+    "VidoreTabfquadRetrieval",
+    "VidoreTatdqaRetrieval",
+]
+VIDORE_V2_MTEB_NAMES = [
+    "Vidore2BioMedicalLecturesRetrieval",
+    "Vidore2EconomicsReportsRetrieval",
+    "Vidore2ESGReportsHLRetrieval",
+    "Vidore2ESGReportsRetrieval",
+]
+DEPRECATED_VIDORE_DATASETS_KEYWORDS = [
     "arxivqa",
     "docvqa",
     "infovqa",
@@ -11,15 +29,48 @@ VIDORE_DATASETS_KEYWORDS = [
     "healthcare_industry",
 ]
 
-VIDORE_2_DATASETS_KEYWORDS = [
+DEPRECATED_VIDORE_2_DATASETS_KEYWORDS = [
     "restaurant_esg",
     "rse_restaurant",
     "mit_biomedical",
     "economics_macro",
 ]
 
-
 def get_datasets_nickname(dataset_name) -> str:
+    if dataset_name == "VidoreArxivQARetrieval":
+        return "ArxivQA"
+    elif dataset_name == "VidoreDocVQARetrieval":
+        return "DocVQA"
+    elif dataset_name == "VidoreInfoVQARetrieval":
+        return "InfoVQA"
+    elif dataset_name == "VidoreTabfquadRetrieval":
+        return "TabFQuad"
+    elif dataset_name == "VidoreTatdqaRetrieval":
+        return "TAT-DQA"
+    elif dataset_name == "VidoreShiftProjectRetrieval":
+        return "Shift Project"
+    elif dataset_name == "VidoreSyntheticDocQAAIRetrieval":
+        return "Artificial Intelligence"
+    elif dataset_name == "VidoreSyntheticDocQAEnergyRetrieval":
+        return "Energy"
+    elif dataset_name == "VidoreSyntheticDocQAGovernmentReportsRetrieval":
+        return "Government Reports"
+    elif dataset_name == "VidoreSyntheticDocQAHealthcareIndustryRetrieval":
+        return "Healthcare Industry"
+
+    elif dataset_name == "Vidore2ESGReportsHLRetrieval":
+        return "ESG Restaurant Human English"
+    elif dataset_name == "Vidore2ESGReportsRetrieval":
+        return "ESG Restaurant Synthetic Multilingual"
+    elif dataset_name == "Vidore2BioMedicalLecturesRetrieval":
+        return "MIT Biomedical Multilingual"
+    elif dataset_name == "Vidore2EconomicsReportsRetrieval":
+        return "Economics Macro Multilingual"
+
+    else:
+        raise ValueError(f"Dataset {dataset_name} not found in ViDoRe")
+
+def deprecated_get_datasets_nickname(dataset_name) -> str:
     if "arxivqa" in dataset_name:
         return "ArxivQA"
 

data/deprecated_model_handler.py

@@ -0,0 +1,124 @@
+import json
+import os
+from typing import Any, Dict
+
+import pandas as pd
+from huggingface_hub import HfApi, hf_hub_download, metadata_load
+
+from .dataset_handler import DEPRECATED_VIDORE_2_DATASETS_KEYWORDS, DEPRECATED_VIDORE_DATASETS_KEYWORDS, deprecated_get_datasets_nickname
+
+BLOCKLIST = ["impactframes"]
+
+
+class DeprecatedModelHandler:
+    def __init__(self, model_infos_path="model_infos.json"):
+        self.api = HfApi()
+        self.model_infos_path = model_infos_path
+        self.model_infos = self._load_model_infos()
+
+    def _load_model_infos(self) -> Dict:
+        if os.path.exists(self.model_infos_path):
+            with open(self.model_infos_path) as f:
+                return json.load(f)
+        return {}
+
+    def _save_model_infos(self):
+        with open(self.model_infos_path, "w") as f:
+            json.dump(self.model_infos, f)
+
+    def _are_results_in_new_vidore_format(self, results: Dict[str, Any]) -> bool:
+        return "metadata" in results and "metrics" in results
+
+    def _is_baseline_repo(self, repo_id: str) -> bool:
+        return repo_id == "vidore/baseline-results"
+
+    def sanitize_model_name(self, model_name):
+        return model_name.replace("/", "_").replace(".", "-thisisapoint-")
+
+    def fuze_model_infos(self, model_name, results):
+        for dataset, metrics in results.items():
+            if dataset not in self.model_infos[model_name]["results"].keys():
+                self.model_infos[model_name]["results"][dataset] = metrics
+            else:
+                continue
+
+    def get_vidore_data(self, metric="ndcg_at_5"):
+        models = self.api.list_models(filter="vidore")
+        repositories = [model.modelId for model in models]  # type: ignore
+
+        # Sort repositories to process non-baseline repos first (to prioritize their results)
+        repositories.sort(key=lambda x: self._is_baseline_repo(x))
+
+        for repo_id in repositories:
+            org_name = repo_id.split("/")[0]
+            if org_name in BLOCKLIST:
+                continue
+            files = [f for f in self.api.list_repo_files(repo_id) if f.endswith("_metrics.json") or f == "results.json"]
+
+            if len(files) == 0:
+                continue
+            else:
+                for file in files:
+                    if file.endswith("results.json"):
+                        model_name = repo_id.replace("/", "_").replace(".", "-thisisapoint-")
+                    else:
+                        model_name = file.split("_metrics.json")[0]
+                        model_name = model_name.replace("/", "_").replace(".", "-thisisapoint-")
+
+                    # Skip if the model is from baseline and we already have results
+
+                    readme_path = hf_hub_download(repo_id, filename="README.md")
+                    meta = metadata_load(readme_path)
+                    try:
+                        result_path = hf_hub_download(repo_id, filename=file)
+
+                        with open(result_path) as f:
+                            results = json.load(f)
+
+                        if self._are_results_in_new_vidore_format(results):
+                            metadata = results["metadata"]
+                            results = results["metrics"]
+
+                        # Handles the case where the model is both in baseline and outside of it
+                        # (prioritizes the non-baseline results)
+                        if self._is_baseline_repo(repo_id) and self.sanitize_model_name(model_name) in self.model_infos:
+                            self.fuze_model_infos(model_name, results)
+
+                        self.model_infos[model_name] = {"meta": meta, "results": results}
+                    except Exception as e:
+                        print(f"Error loading {model_name} - {e}")
+                        continue
+
+    # In order to keep only models relevant to a benchmark
+    def filter_models_by_benchmark(self, benchmark_version=1):
+        filtered_model_infos = {}
+        keywords = DEPRECATED_VIDORE_DATASETS_KEYWORDS if benchmark_version == 1 else DEPRECATED_VIDORE_2_DATASETS_KEYWORDS
+
+        for model, info in self.model_infos.items():
+            results = info["results"]
+            if any(any(keyword in dataset for keyword in keywords) for dataset in results.keys()):
+                filtered_model_infos[model] = info
+
+        return filtered_model_infos
+
+    # Compute the average of a metric for each model,
+    def render_df(self, metric="ndcg_at_5", benchmark_version=1):
+        model_res = {}
+        filtered_model_infos = self.filter_models_by_benchmark(benchmark_version)
+        if len(filtered_model_infos) > 0:
+            for model in filtered_model_infos.keys():
+                res = filtered_model_infos[model]["results"]
+                dataset_res = {}
+                keywords = DEPRECATED_VIDORE_DATASETS_KEYWORDS if benchmark_version == 1 else DEPRECATED_VIDORE_2_DATASETS_KEYWORDS
+                for dataset in res.keys():
+                    if not any(keyword in dataset for keyword in keywords):
+                        continue
+
+                    dataset_nickname = deprecated_get_datasets_nickname(dataset)
+                    dataset_res[dataset_nickname] = res[dataset][metric]
+                model_res[model] = dataset_res
+
+            df = pd.DataFrame(model_res).T
+
+            return df
+        return pd.DataFrame()

data/model_handler.py

@@ -1,98 +1,66 @@
-import json
+from git import Repo
+import shutil
 import os
-from typing import Any, Dict
-
+import json
 import pandas as pd
-from huggingface_hub import HfApi, hf_hub_download, metadata_load
-
-from .dataset_handler import VIDORE_2_DATASETS_KEYWORDS, VIDORE_DATASETS_KEYWORDS, get_datasets_nickname
-
-BLOCKLIST = ["impactframes"]
 
+from .dataset_handler import VIDORE_V1_MTEB_NAMES, VIDORE_V2_MTEB_NAMES, get_datasets_nickname
 
 class ModelHandler:
-    def __init__(self, model_infos_path="model_infos.json"):
-        self.api = HfApi()
-        self.model_infos_path = model_infos_path
-        self.model_infos = self._load_model_infos()
-
-    def _load_model_infos(self) -> Dict:
-        if os.path.exists(self.model_infos_path):
-            with open(self.model_infos_path) as f:
-                return json.load(f)
-        return {}
-
-    def _save_model_infos(self):
-        with open(self.model_infos_path, "w") as f:
-            json.dump(self.model_infos, f)
-
-    def _are_results_in_new_vidore_format(self, results: Dict[str, Any]) -> bool:
-        return "metadata" in results and "metrics" in results
 
-    def _is_baseline_repo(self, repo_id: str) -> bool:
-        return repo_id == "vidore/baseline-results"
+    def __init__(self):
+        self.model_infos = {}
 
-    def sanitize_model_name(self, model_name):
-        return model_name.replace("/", "_").replace(".", "-thisisapoint-")
-
-    def fuze_model_infos(self, model_name, results):
-        for dataset, metrics in results.items():
-            if dataset not in self.model_infos[model_name]["results"].keys():
-                self.model_infos[model_name]["results"][dataset] = metrics
-            else:
-                continue
+    @staticmethod
+    def get_folders(dir_path):
+        return sorted([
+            path_
+            for path_ in os.listdir(dir_path)
+            if os.path.isdir(os.path.join(dir_path, path_))
+        ])
 
     def get_vidore_data(self, metric="ndcg_at_5"):
-        models = self.api.list_models(filter="vidore")
-        repositories = [model.modelId for model in models]  # type: ignore
-
-        # Sort repositories to process non-baseline repos first (to prioritize their results)
-        repositories.sort(key=lambda x: self._is_baseline_repo(x))
-
-        for repo_id in repositories:
-            org_name = repo_id.split("/")[0]
-            if org_name in BLOCKLIST:
-                continue
-            files = [f for f in self.api.list_repo_files(repo_id) if f.endswith("_metrics.json") or f == "results.json"]
-
-            if len(files) == 0:
-                continue
+        repo_url = "https://github.com/embeddings-benchmark/results.git"
+        local_path = "./results"
+        folder_of_interest = "results"
+
+        if os.path.exists(local_path):
+            repo = Repo(local_path)
+            origin = repo.remotes.origin
+            origin.pull()
+        else:
+            Repo.clone_from(repo_url, local_path, depth=1)
+
+        model_names = self.get_folders(os.path.join(local_path, folder_of_interest))
+        for model_name in model_names:
+            revisions = self.get_folders(os.path.join(local_path, folder_of_interest, model_name))
+            first_revision = revisions[0]
+            result_filenames = [
+                result_filename
+                for result_filename in os.listdir(os.path.join(local_path, folder_of_interest, model_name, first_revision))
+                # if result_filename.endswith(".json") and result_filename != "model_meta.json"
+            ]
+            if "model_meta.json" in result_filenames:
+                with open(os.path.join(local_path, folder_of_interest, model_name, first_revision, "model_meta.json"), "r") as f:
+                    meta = json.load(f)
             else:
-                for file in files:
-                    if file.endswith("results.json"):
-                        model_name = repo_id.replace("/", "_").replace(".", "-thisisapoint-")
-                    else:
-                        model_name = file.split("_metrics.json")[0]
-                        model_name = model_name.replace("/", "_").replace(".", "-thisisapoint-")
-
-                    # Skip if the model is from baseline and we already have results
-
-                    readme_path = hf_hub_download(repo_id, filename="README.md")
-                    meta = metadata_load(readme_path)
-                    try:
-                        result_path = hf_hub_download(repo_id, filename=file)
-
-                        with open(result_path) as f:
-                            results = json.load(f)
-
-                        if self._are_results_in_new_vidore_format(results):
-                            metadata = results["metadata"]
-                            results = results["metrics"]
-
-                        # Handles the case where the model is both in baseline and outside of it
-                        # (prioritizes the non-baseline results)
-                        if self._is_baseline_repo(repo_id) and self.sanitize_model_name(model_name) in self.model_infos:
-                            self.fuze_model_infos(model_name, results)
-
-                        self.model_infos[model_name] = {"meta": meta, "results": results}
-                    except Exception as e:
-                        print(f"Error loading {model_name} - {e}")
-                        continue
+                meta = {}
+            results = {}
+            if all(f"{v1_dataset_name}.json" in result_filenames for v1_dataset_name in VIDORE_V1_MTEB_NAMES):
+                for v1_dataset_name in VIDORE_V1_MTEB_NAMES:
+                    with open(os.path.join(local_path, folder_of_interest, model_name, first_revision, f"{v1_dataset_name}.json"), "r") as f:
+                        results[v1_dataset_name] = json.load(f)
+            if all(f"{v2_dataset_name}.json" in result_filenames for v2_dataset_name in VIDORE_V2_MTEB_NAMES):
+                for v2_dataset_name in VIDORE_V2_MTEB_NAMES:
+                    with open(os.path.join(local_path, folder_of_interest, model_name, first_revision, f"{v2_dataset_name}.json"), "r") as f:
+                        results[v2_dataset_name] = json.load(f)
+                if model_name not in self.model_infos:
+                    self.model_infos[model_name] = {}
+                self.model_infos[model_name] = {"meta": meta, "results": results}
 
-    # In order to keep only models relevant to a benchmark
     def filter_models_by_benchmark(self, benchmark_version=1):
         filtered_model_infos = {}
-        keywords = VIDORE_DATASETS_KEYWORDS if benchmark_version == 1 else VIDORE_2_DATASETS_KEYWORDS
+        keywords = VIDORE_V1_MTEB_NAMES if benchmark_version == 1 else VIDORE_V2_MTEB_NAMES
 
         for model, info in self.model_infos.items():
             results = info["results"]
@@ -101,52 +69,26 @@ class ModelHandler:
 
         return filtered_model_infos
 
-    # Compute the average of a metric for each model,
-    def compute_averages(self, metric="ndcg_at_5", benchmark_version=1):
+    def render_df(self, metric="ndcg_at_5", benchmark_version=1):
         model_res = {}
         filtered_model_infos = self.filter_models_by_benchmark(benchmark_version)
         if len(filtered_model_infos) > 0:
             for model in filtered_model_infos.keys():
                 res = filtered_model_infos[model]["results"]
                 dataset_res = {}
-                keywords = VIDORE_DATASETS_KEYWORDS if benchmark_version == 1 else VIDORE_2_DATASETS_KEYWORDS
+                keywords = VIDORE_V1_MTEB_NAMES if benchmark_version == 1 else VIDORE_V2_MTEB_NAMES
+                if "n_parameters" in filtered_model_infos[model]["meta"]:
+                    dataset_res["Model Size (Million Parameters)"] = filtered_model_infos[model]["meta"]["n_parameters"] // 1_000_000
+                else:
+                    dataset_res["Model Size (Million Parameters)"] = None
                 for dataset in res.keys():
                     if not any(keyword in dataset for keyword in keywords):
                         continue
-
                     dataset_nickname = get_datasets_nickname(dataset)
-                    dataset_res[dataset_nickname] = res[dataset][metric]
+                    dataset_res[dataset_nickname] = res[dataset]["scores"]["test"][0][metric]
                 model_res[model] = dataset_res
 
             df = pd.DataFrame(model_res).T
 
             return df
         return pd.DataFrame()
-
-    @staticmethod
-    def add_rank(df, benchmark_version=1):
-        df.fillna(0.0, inplace=True)
-        cols_to_rank = [
-            col
-            for col in df.columns
-            if col
-            not in [
-                "Model",
-                "Model Size (Million Parameters)",
-                "Memory Usage (GB, fp32)",
-                "Embedding Dimensions",
-                "Max Tokens",
-            ]
-        ]
-
-        if len(cols_to_rank) == 1:
-            df.sort_values(cols_to_rank[0], ascending=False, inplace=True)
-        else:
-            df.insert(len(df.columns) - len(cols_to_rank), "Average", df[cols_to_rank].mean(axis=1, skipna=False))
-            df.sort_values("Average", ascending=False, inplace=True)
-        df.insert(0, "Rank", list(range(1, len(df) + 1)))
-        # multiply values by 100 if they are floats and round to 1 decimal place
-        for col in df.columns:
-            if df[col].dtype == "float64":
-                df[col] = df[col].apply(lambda x: round(x * 100, 1))
-        return df

requirements.txt

@@ -0,0 +1,2 @@
+gitpython
+gradio