Add eval caching

app.py

@@ -8,6 +8,8 @@ from datasets import get_dataset_config_names
 from dotenv import load_dotenv
 from huggingface_hub import list_datasets
 
+from evaluation import (EvaluationInfo, compute_evaluation_id,
+                        get_evaluation_ids)
 from utils import (get_compatible_models, get_key, get_metadata, http_get,
                    http_post)
 
@@ -244,6 +246,24 @@ with st.form(key="form"):
 
     selected_models = st.multiselect("Select the models you wish to evaluate", compatible_models)
     print("Selected models:", selected_models)
+
+    evaluation_ids = get_evaluation_ids()
+
+    for idx, model in enumerate(selected_models):
+        eval_info = EvaluationInfo(
+            task=selected_task,
+            model=model,
+            dataset_name=selected_dataset,
+            dataset_config=selected_config,
+            dataset_split=selected_split,
+        )
+        candidate_id = hash(eval_info)
+        if candidate_id in evaluation_ids:
+            st.info(f"Model {model} has already been evaluated on this configuration. Skipping ...")
+            selected_models.pop(idx)
+
+    print("Selected models:", selected_models)
+
     submit_button = st.form_submit_button("Make submission")
 
     if submit_button:

evaluation.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+
+from huggingface_hub import DatasetFilter, HfApi
+from huggingface_hub.hf_api import DatasetInfo
+
+
+@dataclass(frozen=True, eq=True)
+class EvaluationInfo:
+    task: str
+    model: str
+    dataset_name: str
+    dataset_config: str
+    dataset_split: str
+
+
+def compute_evaluation_id(dataset_info: DatasetInfo) -> int:
+    metadata = dataset_info.cardData["eval_info"]
+    metadata.pop("col_mapping", None)
+    evaluation_info = EvaluationInfo(**metadata)
+    return hash(evaluation_info)
+
+
+def get_evaluation_ids():
+    filt = DatasetFilter(author="autoevaluate")
+    evaluation_datasets = HfApi().list_datasets(filter=filt, full=True)
+    return [compute_evaluation_id(dset) for dset in evaluation_datasets]

utils.py

@@ -1,7 +1,7 @@
 from typing import Dict, Union
 
 import requests
-from huggingface_hub import DatasetFilter, HfApi, ModelFilter
+from huggingface_hub import HfApi, ModelFilter
 
 AUTOTRAIN_TASK_TO_HUB_TASK = {
     "binary_classification": "text-classification",