update to latest

README.md

@@ -4,10 +4,11 @@ emoji: 📉
 colorFrom: green
 colorTo: indigo
 sdk: gradio
-sdk_version: 3.27.0
+sdk_version: 3.43.2
 app_file: app.py
 pinned: true
 license: apache-2.0
+duplicated_from: HuggingFaceH4/open_llm_leaderboard
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -222,21 +222,6 @@ def add_new_eval(
 
 
 # Basics
-def refresh() -> list[pd.DataFrame]:
-    leaderboard_df = get_leaderboard_df(eval_results, eval_results_private, COLS, BENCHMARK_COLS)
-    (
-        finished_eval_queue_df,
-        running_eval_queue_df,
-        pending_eval_queue_df,
-    ) = get_evaluation_queue_df(eval_queue, eval_queue_private, EVAL_REQUESTS_PATH, EVAL_COLS)
-    return (
-        leaderboard_df,
-        finished_eval_queue_df,
-        running_eval_queue_df,
-        pending_eval_queue_df,
-    )
-
-
 def change_tab(query_param: str):
     query_param = query_param.replace("'", '"')
     query_param = json.loads(query_param)
@@ -248,17 +233,16 @@ def change_tab(query_param: str):
 
 
 # Searching and filtering
-def search_table(df: pd.DataFrame, current_columns_df: pd.DataFrame, query: str) -> pd.DataFrame:
-    current_columns = current_columns_df.columns
-    if AutoEvalColumn.model_type.name in current_columns:
-        filtered_df = df[
-            (df[AutoEvalColumn.dummy.name].str.contains(query, case=False))
-            | (df[AutoEvalColumn.model_type.name].str.contains(query, case=False))
-        ]
-    else:
-        filtered_df = df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
-    return filtered_df[current_columns]
+def update_table(hidden_df: pd.DataFrame, current_columns_df: pd.DataFrame, columns: list, type_query: list, precision_query: str, size_query: list, show_deleted: bool, query: str):
+    filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
+    if query != "":
+        filtered_df = search_table(filtered_df, query)
+    df = select_columns(filtered_df, columns)
+
+    return df
 
+def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
+    return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
 
 def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     always_here_cols = [
@@ -272,31 +256,32 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     return filtered_df
 
 NUMERIC_INTERVALS = {
-    "< 1.5B": (0, 1.5),
-    "~3B": (1.5, 5),
-    "~7B": (6, 11),
-    "~13B": (12, 15),
-    # "~35B": (16, 55),
-    # "60B+": (55, 10000),
+    "Unknown": pd.Interval(-1, 0, closed="right"), 
+    "< 1.5B": pd.Interval(0, 1.5, closed="right"),
+    "~3B": pd.Interval(1.5, 5, closed="right"),
+    "~7B": pd.Interval(6, 11, closed="right"),
+    "~13B": pd.Interval(12, 15, closed="right"),
+    # "~35B": pd.Interval(16, 55, closed="right"),
+    # "60B+": pd.Interval(55, 10000, closed="right"),
 }
 
 def filter_models(
-    df: pd.DataFrame, current_columns_df: pd.DataFrame, type_query: list, size_query: list, show_deleted: bool
+    df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
 ) -> pd.DataFrame:
-    current_columns = current_columns_df.columns
-
     # Show all models
     if show_deleted:
-        filtered_df = df[current_columns]
+        filtered_df = df
     else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True][current_columns]
+        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
 
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+    filtered_df = filtered_df[df[AutoEvalColumn.precision.name].isin(precision_query)]
 
-    numeric_interval = [NUMERIC_INTERVALS[s] for s in size_query]
+    numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
-    filtered_df = filtered_df[params_column.between(numeric_interval[0][0], numeric_interval[-1][1])]
+    mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
+    filtered_df = filtered_df.loc[mask]
 
     return filtered_df
 
@@ -310,6 +295,12 @@ with demo:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
                 with gr.Column():
+                    with gr.Row():
+                        search_bar = gr.Textbox(
+                            placeholder=" 🔍 Search for your model and press ENTER...",
+                            show_label=False,
+                            elem_id="search-bar",
+                        )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
                             choices=[
@@ -343,11 +334,6 @@ with demo:
                             value=True, label="👀 Show gated/private/deleted models", interactive=True
                         )
                 with gr.Column(min_width=320):
-                    search_bar = gr.Textbox(
-                        placeholder="🔍 Search for your model and press ENTER...",
-                        show_label=False,
-                        elem_id="search-bar",
-                    )
                     with gr.Box(elem_id="box-filter"):
                         filter_columns_type = gr.CheckboxGroup(
                             label="Model types",
@@ -366,6 +352,13 @@ with demo:
                             interactive=True,
                             elem_id="filter-columns-type",
                         )
+                        filter_columns_precision = gr.CheckboxGroup(
+                            label="Precision",
+                            choices=["torch.float16"], #, "torch.bfloat16", "torch.float32", "8bit", "4bit", "GPTQ"],
+                            value=["torch.float16"], #, "torch.bfloat16", "torch.float32", "8bit", "4bit", "GPTQ"],
+                            interactive=False,
+                            elem_id="filter-columns-precision",
+                        )
                         filter_columns_size = gr.CheckboxGroup(
                             label="Model sizes",
                             choices=list(NUMERIC_INTERVALS.keys()),
@@ -402,55 +395,93 @@ with demo:
                 visible=False,
             )
             search_bar.submit(
-                search_table,
+                update_table,
                 [
                     hidden_leaderboard_table_for_search,
                     leaderboard_table,
+                    shown_columns,
+                    filter_columns_type,
+                    filter_columns_precision,
+                    filter_columns_size,
+                    deleted_models_visibility,
                     search_bar,
                 ],
                 leaderboard_table,
             )
             shown_columns.change(
-                select_columns,
-                [hidden_leaderboard_table_for_search, shown_columns],
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    shown_columns,
+                    filter_columns_type,
+                    filter_columns_precision,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                    search_bar,
+                ],
                 leaderboard_table,
-                queue=False,
+                queue=True,
             )
             filter_columns_type.change(
-                filter_models,
+                update_table,
                 [
                     hidden_leaderboard_table_for_search,
                     leaderboard_table,
+                    shown_columns,
                     filter_columns_type,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
+                    search_bar,
                 ],
                 leaderboard_table,
-                queue=False,
+                queue=True,
+            )
+            filter_columns_precision.change(
+                update_table,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    shown_columns,
+                    filter_columns_type,
+                    filter_columns_precision,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                    search_bar,
+                ],
+                leaderboard_table,
+                queue=True,
             )
             filter_columns_size.change(
-                filter_models,
+                update_table,
                 [
                     hidden_leaderboard_table_for_search,
                     leaderboard_table,
+                    shown_columns,
                     filter_columns_type,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
+                    search_bar,
                 ],
                 leaderboard_table,
-                queue=False,
+                queue=True,
             )
             deleted_models_visibility.change(
-                filter_models,
+                update_table,
                 [
                     hidden_leaderboard_table_for_search,
                     leaderboard_table,
+                    shown_columns,
                     filter_columns_type,
+                    filter_columns_precision,
                     filter_columns_size,
                     deleted_models_visibility,
+                    search_bar,
                 ],
                 leaderboard_table,
-                queue=False,
+                queue=True,
             )
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
@@ -556,20 +587,6 @@ with demo:
                 submission_result,
             )
 
-        with gr.Row():
-            refresh_button = gr.Button("Refresh")
-            refresh_button.click(
-                refresh,
-                inputs=[],
-                outputs=[
-                    leaderboard_table,
-                    finished_eval_table,
-                    running_eval_table,
-                    pending_eval_table,
-                ],
-                api_name='refresh'
-            )
-
     with gr.Row():
         with gr.Accordion("📙 Citation", open=False):
             citation_button = gr.Textbox(
@@ -589,6 +606,6 @@ with demo:
     )
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=3600)
+scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
 demo.queue(concurrency_count=40).launch()

requirements.txt

@@ -1,3 +1,4 @@
+accelerate==0.23.0
 aiofiles==23.1.0
 aiohttp==3.8.4
 aiosignal==1.3.1
@@ -19,8 +20,8 @@ filelock==3.11.0
 fonttools==4.39.3
 frozenlist==1.3.3
 fsspec==2023.4.0
-gradio==3.27.0
-gradio_client==0.1.3
+gradio==3.43.2
+gradio-client==0.5.0
 h11==0.14.0
 httpcore==0.17.0
 httpx==0.24.0
@@ -59,7 +60,7 @@ sniffio==1.3.0
 starlette==0.26.1
 toolz==0.12.0
 tqdm==4.65.0
-transformers==4.32.0
+transformers==4.33.1
 typing_extensions==4.5.0
 tzdata==2023.3
 tzlocal==4.3

src/display_models/get_model_metadata.py

@@ -2,11 +2,14 @@ import glob
 import json
 import os
 import re
+import pickle
 from typing import List
 
 import huggingface_hub
 from huggingface_hub import HfApi
 from tqdm import tqdm
+from transformers import AutoModel, AutoConfig
+from accelerate import init_empty_weights
 
 from src.display_models.model_metadata_flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
 from src.display_models.model_metadata_type import MODEL_TYPE_METADATA, ModelType, model_type_from_str
@@ -16,27 +19,53 @@ api = HfApi(token=os.environ.get("H4_TOKEN", None))
 
 
 def get_model_infos_from_hub(leaderboard_data: List[dict]):
+    # load cache from disk
+    try:
+        with open("model_info_cache.pkl", "rb") as f:
+            model_info_cache = pickle.load(f)
+    except (EOFError, FileNotFoundError):
+        model_info_cache = {}
+    try:
+        with open("model_size_cache.pkl", "rb") as f:
+            model_size_cache = pickle.load(f)
+    except (EOFError, FileNotFoundError):
+        model_size_cache = {}
+
     for model_data in tqdm(leaderboard_data):
         model_name = model_data["model_name_for_query"]
-        try:
-            model_info = api.model_info(model_name)
-        except huggingface_hub.utils._errors.RepositoryNotFoundError:
-            print("Repo not found!", model_name)
-            model_data[AutoEvalColumn.license.name] = None
-            model_data[AutoEvalColumn.likes.name] = None
-            model_data[AutoEvalColumn.params.name] = get_model_size(model_name, None)
-            continue
+
+        if model_name in model_info_cache:
+            model_info = model_info_cache[model_name]
+        else:
+            try:
+                model_info = api.model_info(model_name)
+                model_info_cache[model_name] = model_info
+            except huggingface_hub.utils._errors.RepositoryNotFoundError:
+                print("Repo not found!", model_name)
+                model_data[AutoEvalColumn.license.name] = None
+                model_data[AutoEvalColumn.likes.name] = None
+                if model_name not in model_size_cache:
+                    model_size_cache[model_name] = get_model_size(model_name, None)
+                model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
 
         model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
         model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
-        model_data[AutoEvalColumn.params.name] = get_model_size(model_name, model_info)
+        if model_name not in model_size_cache:
+            model_size_cache[model_name] = get_model_size(model_name, model_info)
+        model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
+    
+    # save cache to disk in pickle format
+    with open("model_info_cache.pkl", "wb") as f:
+        pickle.dump(model_info_cache, f)
+    with open("model_size_cache.pkl", "wb") as f:
+        pickle.dump(model_size_cache, f)
 
 
 def get_model_license(model_info):
     try:
         return model_info.cardData["license"]
     except Exception:
-        return None
+        return "?"
 
 
 def get_model_likes(model_info):
@@ -52,11 +81,17 @@ def get_model_size(model_name, model_info):
         return round(model_info.safetensors["total"] / 1e9, 3)
     except AttributeError:
         try:
-            size_match = re.search(size_pattern, model_name.lower())
-            size = size_match.group(0)
-            return round(float(size[:-1]) if size[-1] == "b" else float(size[:-1]) / 1e3, 3)
-        except AttributeError:
-            return None
+            config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)
+            with init_empty_weights():
+                model = AutoModel.from_config(config, trust_remote_code=False)
+            return round(sum(p.numel() for p in model.parameters() if p.requires_grad) / 1e9, 3)
+        except (EnvironmentError, ValueError): # model config not found, likely private  
+            try:
+                size_match = re.search(size_pattern, model_name.lower())
+                size = size_match.group(0)
+                return round(float(size[:-1]) if size[-1] == "b" else float(size[:-1]) / 1e3, 3)
+            except AttributeError:
+                return 0
 
 
 def get_model_type(leaderboard_data: List[dict]):

src/display_models/model_metadata_flags.py

@@ -1,15 +1,8 @@
 # Models which have been flagged by users as being problematic for a reason or another
 # (Model name to forum discussion link)
 FLAGGED_MODELS = {
-    "Voicelab/trurl-2-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/202",
-    "deepnight-research/llama-2-70B-inst": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/207",
-    "Aspik101/trurl-2-13b-pl-instruct_unload": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/213",
-    "Fredithefish/ReasonixPajama-3B-HF": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/236",
-    "TigerResearch/tigerbot-7b-sft-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/237",
-    "gaodrew/gaodrew-gorgonzola-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/215",
 }
 
 # Models which have been requested by orgs to not be submitted on the leaderboard
 DO_NOT_SUBMIT_MODELS = [
-    "Voicelab/trurl-2-13b",  # trained on MMLU
 ]

src/display_models/read_results.py

@@ -87,11 +87,11 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
     if len(model_split) == 1:
         org = None
         model = model_split[0]
-        result_key = f"{model}_{model_sha}_{precision}"
+        result_key = f"{model}_{precision}"
     else:
         org = model_split[0]
         model = model_split[1]
-        result_key = f"{org}_{model}_{model_sha}_{precision}"
+        result_key = f"{org}_{model}_{precision}"
 
     eval_results = []
     for benchmark, metric in zip(BENCHMARKS, METRICS):

src/load_from_hub.py

@@ -80,11 +80,8 @@ def get_leaderboard_df(
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
 
-
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
-
-    print(df)
     return df
 
 
@@ -125,7 +122,7 @@ def get_evaluation_queue_df(
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
     df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)