Merge pull request #10 from huggingface/add-caching

.github/workflows/quality.yml

@@ -0,0 +1,29 @@
+name: Code quality
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+
+  check_code_quality:
+    name: Check code quality
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup Python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install black isort flake8
+      - name: Code quality
+        run: |
+          make quality

Makefile

@@ -0,0 +1,8 @@
+style:
+	python -m black --line-length 119 --target-version py39 .
+	python -m isort .
+
+quality:
+	python -m black --check --line-length 119 --target-version py39 .
+	python -m isort --check-only .
+	python -m flake8 --max-line-length 119

app.py

@@ -8,8 +8,8 @@ from datasets import get_dataset_config_names
 from dotenv import load_dotenv
 from huggingface_hub import list_datasets
 
-from utils import (get_compatible_models, get_key, get_metadata, http_get,
-                   http_post)
+from evaluation import filter_evaluated_models
+from utils import get_compatible_models, get_key, get_metadata, http_get, http_post
 
 if Path(".env").is_file():
     load_dotenv(".env")
@@ -33,9 +33,9 @@ TASK_TO_ID = {
 SUPPORTED_TASKS = list(TASK_TO_ID.keys())
 
 
-###########
-### APP ###
-###########
+#######
+# APP #
+#######
 st.title("Evaluation as a Service")
 st.markdown(
     """
@@ -64,18 +64,22 @@ if metadata is None:
     st.warning("No evaluation metadata found. Please configure the evaluation job below.")
 
 with st.expander("Advanced configuration"):
-    ## Select task
+    # Select task
     selected_task = st.selectbox(
         "Select a task",
         SUPPORTED_TASKS,
         index=SUPPORTED_TASKS.index(metadata[0]["task_id"]) if metadata is not None else 0,
     )
-    ### Select config
+    # Select config
     configs = get_dataset_config_names(selected_dataset)
     selected_config = st.selectbox("Select a config", configs)
 
-    ## Select splits
-    splits_resp = http_get(path="/splits", domain=DATASETS_PREVIEW_API, params={"dataset": selected_dataset})
+    # Select splits
+    splits_resp = http_get(
+        path="/splits",
+        domain=DATASETS_PREVIEW_API,
+        params={"dataset": selected_dataset},
+    )
     if splits_resp.status_code == 200:
         split_names = []
         all_splits = splits_resp.json()
@@ -89,11 +93,15 @@ with st.expander("Advanced configuration"):
             index=split_names.index(metadata[0]["splits"]["eval_split"]) if metadata is not None else 0,
         )
 
-    ## Select columns
+    # Select columns
     rows_resp = http_get(
         path="/rows",
         domain=DATASETS_PREVIEW_API,
-        params={"dataset": selected_dataset, "config": selected_config, "split": selected_split},
+        params={
+            "dataset": selected_dataset,
+            "config": selected_config,
+            "split": selected_split,
+        },
     ).json()
     col_names = list(pd.json_normalize(rows_resp["rows"][0]["row"]).columns)
 
@@ -135,7 +143,7 @@ with st.expander("Advanced configuration"):
             st.markdown("`tags` column")
         with col2:
             tokens_col = st.selectbox(
-                "This column should contain the parts of the text (as an array of tokens) you want to assign labels to",
+                "This column should contain the array of tokens",
                 col_names,
                 index=col_names.index(get_key(metadata[0]["col_mapping"], "tokens")) if metadata is not None else 0,
             )
@@ -244,65 +252,86 @@ with st.form(key="form"):
 
     selected_models = st.multiselect("Select the models you wish to evaluate", compatible_models)
     print("Selected models:", selected_models)
+
+    selected_models = filter_evaluated_models(
+        selected_models,
+        selected_task,
+        selected_dataset,
+        selected_config,
+        selected_split,
+    )
+    print("Selected models:", selected_models)
+
     submit_button = st.form_submit_button("Make submission")
 
     if submit_button:
-        project_id = str(uuid.uuid4())[:3]
-        payload = {
-            "username": AUTOTRAIN_USERNAME,
-            "proj_name": f"my-eval-project-{project_id}",
-            "task": TASK_TO_ID[selected_task],
-            "config": {
-                "language": "en",
-                "max_models": 5,
-                "instance": {
-                    "provider": "aws",
-                    "instance_type": "ml.g4dn.4xlarge",
-                    "max_runtime_seconds": 172800,
-                    "num_instances": 1,
-                    "disk_size_gb": 150,
-                },
-                "evaluation": {
-                    "metrics": [],
-                    "models": selected_models,
-                },
-            },
-        }
-        print(f"Payload: {payload}")
-        project_json_resp = http_post(
-            path="/projects/create", payload=payload, token=HF_TOKEN, domain=AUTOTRAIN_BACKEND_API
-        ).json()
-        print(project_json_resp)
-
-        if project_json_resp["created"]:
+        if len(selected_models) > 0:
+            project_id = str(uuid.uuid4())[:3]
             payload = {
-                "split": 4,  # use "auto" split choice in AutoTrain
-                "col_mapping": col_mapping,
-                "load_config": {"max_size_bytes": 0, "shuffle": False},
+                "username": AUTOTRAIN_USERNAME,
+                "proj_name": f"my-eval-project-{project_id}",
+                "task": TASK_TO_ID[selected_task],
+                "config": {
+                    "language": "en",
+                    "max_models": 5,
+                    "instance": {
+                        "provider": "aws",
+                        "instance_type": "ml.g4dn.4xlarge",
+                        "max_runtime_seconds": 172800,
+                        "num_instances": 1,
+                        "disk_size_gb": 150,
+                    },
+                    "evaluation": {
+                        "metrics": [],
+                        "models": selected_models,
+                    },
+                },
             }
-            data_json_resp = http_post(
-                path=f"/projects/{project_json_resp['id']}/data/{selected_dataset}",
+            print(f"Payload: {payload}")
+            project_json_resp = http_post(
+                path="/projects/create",
                 payload=payload,
                 token=HF_TOKEN,
                 domain=AUTOTRAIN_BACKEND_API,
-                params={"type": "dataset", "config_name": selected_config, "split_name": selected_split},
             ).json()
-            print(data_json_resp)
-            if data_json_resp["download_status"] == 1:
-                train_json_resp = http_get(
-                    path=f"/projects/{project_json_resp['id']}/data/start_process",
+            print(project_json_resp)
+
+            if project_json_resp["created"]:
+                payload = {
+                    "split": 4,  # use "auto" split choice in AutoTrain
+                    "col_mapping": col_mapping,
+                    "load_config": {"max_size_bytes": 0, "shuffle": False},
+                }
+                data_json_resp = http_post(
+                    path=f"/projects/{project_json_resp['id']}/data/{selected_dataset}",
+                    payload=payload,
                     token=HF_TOKEN,
                     domain=AUTOTRAIN_BACKEND_API,
+                    params={
+                        "type": "dataset",
+                        "config_name": selected_config,
+                        "split_name": selected_split,
+                    },
                 ).json()
-                print(train_json_resp)
-                if train_json_resp["success"]:
-                    st.success(f"✅ Successfully submitted evaluation job with project ID {project_id}")
-                    st.markdown(
-                        f"""
-                    Evaluation takes appoximately 1 hour to complete, so grab a ☕ or 🍵 while you wait:
+                print(data_json_resp)
+                if data_json_resp["download_status"] == 1:
+                    train_json_resp = http_get(
+                        path=f"/projects/{project_json_resp['id']}/data/start_process",
+                        token=HF_TOKEN,
+                        domain=AUTOTRAIN_BACKEND_API,
+                    ).json()
+                    print(train_json_resp)
+                    if train_json_resp["success"]:
+                        st.success(f"✅ Successfully submitted evaluation job with project ID {project_id}")
+                        st.markdown(
+                            """
+                        Evaluation takes appoximately 1 hour to complete, so grab a ☕ or 🍵 while you wait:
 
-                    * 📊 Click [here](https://huggingface.co/spaces/autoevaluate/leaderboards) to view the results from your submission
-                    """
-                    )
-                else:
-                    st.error("🙈 Oh noes, there was an error submitting your submission!")
+                        * 📊 Click [here](https://huggingface.co/spaces/autoevaluate/leaderboards) to view the \
+                            results from your submission
+                        """
+                        )
+                    else:
+                        st.error("🙈 Oh noes, there was an error submitting your evaluation job!")
+        else:
+            st.warning("⚠️ No models were selected for evaluation!")

evaluation.py

@@ -0,0 +1,46 @@
+from dataclasses import dataclass
+
+import streamlit as st
+from huggingface_hub import DatasetFilter, HfApi
+from huggingface_hub.hf_api import DatasetInfo
+
+
+@dataclass(frozen=True, eq=True)
+class EvaluationInfo:
+    task: str
+    model: str
+    dataset_name: str
+    dataset_config: str
+    dataset_split: str
+
+
+def compute_evaluation_id(dataset_info: DatasetInfo) -> int:
+    metadata = dataset_info.cardData["eval_info"]
+    metadata.pop("col_mapping", None)
+    evaluation_info = EvaluationInfo(**metadata)
+    return hash(evaluation_info)
+
+
+def get_evaluation_ids():
+    filt = DatasetFilter(author="autoevaluate")
+    evaluation_datasets = HfApi().list_datasets(filter=filt, full=True)
+    return [compute_evaluation_id(dset) for dset in evaluation_datasets]
+
+
+def filter_evaluated_models(models, task, dataset_name, dataset_config, dataset_split):
+    evaluation_ids = get_evaluation_ids()
+
+    for idx, model in enumerate(models):
+        evaluation_info = EvaluationInfo(
+            task=task,
+            model=model,
+            dataset_name=dataset_name,
+            dataset_config=dataset_config,
+            dataset_split=dataset_split,
+        )
+        candidate_id = hash(evaluation_info)
+        if candidate_id in evaluation_ids:
+            st.info(f"Model {model} has already been evaluated on this configuration. Skipping evaluation...")
+            models.pop(idx)
+
+    return models

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.isort]
+profile = "black"

utils.py

@@ -1,7 +1,7 @@
 from typing import Dict, Union
 
 import requests
-from huggingface_hub import DatasetFilter, HfApi, ModelFilter
+from huggingface_hub import HfApi, ModelFilter
 
 AUTOTRAIN_TASK_TO_HUB_TASK = {
     "binary_classification": "text-classification",
@@ -27,7 +27,11 @@ def http_post(path: str, token: str, payload=None, domain: str = None, params=No
     """HTTP POST request to the AutoNLP API, raises UnreachableAPIError if the API cannot be reached"""
     try:
         response = requests.post(
-            url=domain + path, json=payload, headers=get_auth_headers(token=token), allow_redirects=True, params=params
+            url=domain + path,
+            json=payload,
+            headers=get_auth_headers(token=token),
+            allow_redirects=True,
+            params=params,
         )
     except requests.exceptions.ConnectionError:
         print("❌ Failed to reach AutoNLP API, check your internet connection")
@@ -39,7 +43,10 @@ def http_get(path: str, domain: str, token: str = None, params: dict = None) ->
     """HTTP POST request to the AutoNLP API, raises UnreachableAPIError if the API cannot be reached"""
     try:
         response = requests.get(
-            url=domain + path, headers=get_auth_headers(token=token), allow_redirects=True, params=params
+            url=domain + path,
+            headers=get_auth_headers(token=token),
+            allow_redirects=True,
+            params=params,
         )
     except requests.exceptions.ConnectionError:
         print("❌ Failed to reach AutoNLP API, check your internet connection")
@@ -58,7 +65,9 @@ def get_metadata(dataset_name: str) -> Union[Dict, None]:
 def get_compatible_models(task, dataset_name):
     # TODO: relax filter on PyTorch models once supported in AutoTrain
     filt = ModelFilter(
-        task=AUTOTRAIN_TASK_TO_HUB_TASK[task], trained_dataset=dataset_name, library=["transformers", "pytorch"]
+        task=AUTOTRAIN_TASK_TO_HUB_TASK[task],
+        trained_dataset=dataset_name,
+        library=["transformers", "pytorch"],
     )
     compatible_models = api.list_models(filter=filt)
     return [model.modelId for model in compatible_models]