update to latest version

.gitattributes

@@ -25,7 +25,6 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
 *.wasm filter=lfs diff=lfs merge=lfs -text

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -1,12 +1,12 @@
 ---
 title: Leaderboard Test
 emoji: 📉
-colorFrom: yellow
-colorTo: red
+colorFrom: green
+colorTo: indigo
 sdk: gradio
 sdk_version: 3.27.0
 app_file: app.py
-pinned: false
+pinned: true
 license: apache-2.0
 ---
 

app.py

@@ -2,23 +2,33 @@ import json
 import os
 from datetime import datetime, timezone
 
-
 import gradio as gr
-import numpy as np
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
-from transformers import AutoConfig
 
-from src.auto_leaderboard.get_model_metadata import apply_metadata
-from src.assets.text_content import *
-from src.auto_leaderboard.load_results import get_eval_results_dicts, make_clickable_model
-from src.assets.hardcoded_evals import gpt4_values, gpt35_values, baseline
 from src.assets.css_html_js import custom_css, get_window_url_params
-from src.utils_display import AutoEvalColumn, EvalQueueColumn, fields, styled_error, styled_warning, styled_message
-from src.init import get_all_requested_models, load_all_info_from_hub
-
-pd.set_option('display.precision', 1)
+from src.assets.text_content import (
+    CITATION_BUTTON_LABEL,
+    CITATION_BUTTON_TEXT,
+    EVALUATION_QUEUE_TEXT,
+    INTRODUCTION_TEXT,
+    LLM_BENCHMARKS_TEXT,
+    TITLE,
+)
+from src.display_models.get_model_metadata import DO_NOT_SUBMIT_MODELS, ModelType
+from src.display_models.utils import (
+    AutoEvalColumn,
+    EvalQueueColumn,
+    fields,
+    styled_error,
+    styled_message,
+    styled_warning,
+)
+from src.load_from_hub import get_evaluation_queue_df, get_leaderboard_df, is_model_on_hub, load_all_info_from_hub
+from src.rate_limiting import user_submission_permission
+
+pd.set_option("display.precision", 1)
 
 # clone / pull the lmeh eval data
 H4_TOKEN = os.environ.get("H4_TOKEN", None)
@@ -37,20 +47,17 @@ EVAL_RESULTS_PATH = "eval-results"
 EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
 EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
 
-api = HfApi()
+api = HfApi(token=H4_TOKEN)
 
-def restart_space():
-    api.restart_space(
-        repo_id="BearSean/leaderboard-test", token=H4_TOKEN
-    )
 
-eval_queue, requested_models, eval_results = load_all_info_from_hub(QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH)
+def restart_space():
+    api.restart_space(repo_id="BearSean/leaderboard-test", token=H4_TOKEN)
 
-if not IS_PUBLIC:
-    eval_queue_private, requested_models_private, eval_results_private = load_all_info_from_hub(PRIVATE_QUEUE_REPO, PRIVATE_RESULTS_REPO, EVAL_REQUESTS_PATH_PRIVATE, EVAL_RESULTS_PATH_PRIVATE)
-else:
-    eval_queue_private, eval_results_private = None, None
+# Rate limit variables
+RATE_LIMIT_PERIOD = 7
+RATE_LIMIT_QUOTA = 5
 
+# Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
@@ -63,116 +70,51 @@ if not IS_PUBLIC:
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [c.name for c in [AutoEvalColumn.arc, AutoEvalColumn.hellaswag, AutoEvalColumn.mmlu, AutoEvalColumn.truthfulqa]]
-
-
-def has_no_nan_values(df, columns):
-    return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
-    return df[columns].isna().any(axis=1)
-
-
-def get_leaderboard_df():
-    if eval_results:
-        print("Pulling evaluation results for the leaderboard.")
-        eval_results.git_pull()
-    if eval_results_private:
-        print("Pulling evaluation results for the leaderboard.")
-        eval_results_private.git_pull()
-
-    all_data = get_eval_results_dicts(IS_PUBLIC)
+BENCHMARK_COLS = [
+    c.name
+    for c in [
+        AutoEvalColumn.arc,
+        AutoEvalColumn.hellaswag,
+        AutoEvalColumn.mmlu,
+        AutoEvalColumn.truthfulqa,
+    ]
+]
 
-    if not IS_PUBLIC:
-        all_data.append(gpt4_values)
-        all_data.append(gpt35_values)
+## LOAD INFO FROM HUB
+eval_queue, requested_models, eval_results, users_to_submission_dates = load_all_info_from_hub(
+    QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH
+)
 
-    all_data.append(baseline)
-    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
+if not IS_PUBLIC:
+    (eval_queue_private, requested_models_private, eval_results_private, _) = load_all_info_from_hub(
+        PRIVATE_QUEUE_REPO,
+        PRIVATE_RESULTS_REPO,
+        EVAL_REQUESTS_PATH_PRIVATE,
+        EVAL_RESULTS_PATH_PRIVATE,
+    )
+else:
+    eval_queue_private, eval_results_private = None, None
 
-    df = pd.DataFrame.from_records(all_data)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[COLS].round(decimals=2)
+original_df = get_leaderboard_df(eval_results, eval_results_private, COLS, BENCHMARK_COLS)
+models = original_df["model_name_for_query"].tolist() # needed for model backlinks in their to the leaderboard
 
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, BENCHMARK_COLS)]
-    return df
+# Commented out because it causes infinite restart loops in local
+# to_be_dumped = f"models = {repr(models)}\n"
 
+# with open("models_backlinks.py", "w") as f:
+#     f.write(to_be_dumped)
 
-def get_evaluation_queue_df():
-    if eval_queue:
-        print("Pulling changes for the evaluation queue.")
-        eval_queue.git_pull()
-    if eval_queue_private:
-        print("Pulling changes for the evaluation queue.")
-        eval_queue_private.git_pull()
+# print(to_be_dumped)
 
-    entries = [
-        entry
-        for entry in os.listdir(EVAL_REQUESTS_PATH)
-        if not entry.startswith(".")
-    ]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(EVAL_REQUESTS_PATH, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            data["# params"] = "unknown"
-            data["model"] = make_clickable_model(data["model"])
-            data["revision"] = data.get("revision", "main")
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [
-                e
-                for e in os.listdir(f"{EVAL_REQUESTS_PATH}/{entry}")
-                if not e.startswith(".")
-            ]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(EVAL_REQUESTS_PATH, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                # data["# params"] = get_n_params(data["model"])
-                data["model"] = make_clickable_model(data["model"])
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=EVAL_COLS)
-    df_running = pd.DataFrame.from_records(running_list, columns=EVAL_COLS)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=EVAL_COLS)
-    return df_finished[EVAL_COLS], df_running[EVAL_COLS], df_pending[EVAL_COLS]
-
-
-
-original_df = get_leaderboard_df()
 leaderboard_df = original_df.copy()
 (
     finished_eval_queue_df,
     running_eval_queue_df,
     pending_eval_queue_df,
-) = get_evaluation_queue_df()
-
-def is_model_on_hub(model_name, revision) -> bool:
-    try:
-        AutoConfig.from_pretrained(model_name, revision=revision)
-        return True, None
-    
-    except ValueError as e:
-        return False, "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard."
-
-    except Exception as e:
-        print(f"Could not get the model config from the hub.: {e}")
-        return False, "was not found on hub!"
+) = get_evaluation_queue_df(eval_queue, eval_queue_private, EVAL_REQUESTS_PATH, EVAL_COLS)
 
 
+## INTERACTION FUNCTIONS
 def add_new_eval(
     model: str,
     base_model: str,
@@ -185,6 +127,14 @@ def add_new_eval(
     precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
+    num_models_submitted_in_period = user_submission_permission(model, users_to_submission_dates, RATE_LIMIT_PERIOD)
+    if num_models_submitted_in_period > RATE_LIMIT_QUOTA:
+        error_msg = f"Organisation or user `{model.split('/')[0]}`"
+        error_msg += f"already has {num_models_submitted_in_period} model requests submitted to the leaderboard "
+        error_msg += f"in the last {RATE_LIMIT_PERIOD} days.\n"
+        error_msg += "Please wait a couple of days before resubmitting, so that everybody can enjoy using the leaderboard 🤗"
+        return styled_error(error_msg)
+
     if model_type is None or model_type == "":
         return styled_error("Please select a model type.")
 
@@ -196,13 +146,12 @@ def add_new_eval(
         base_model_on_hub, error = is_model_on_hub(base_model, revision)
         if not base_model_on_hub:
             return styled_error(f'Base model "{base_model}" {error}')
-        
 
     if not weight_type == "Adapter":
         model_on_hub, error = is_model_on_hub(model, revision)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')
-    
+
     print("adding new eval")
 
     eval_entry = {
@@ -227,8 +176,12 @@ def add_new_eval(
     os.makedirs(OUT_DIR, exist_ok=True)
     out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
 
+    # Check if the model has been forbidden:
+    if out_path.split("eval-queue/")[1] in DO_NOT_SUBMIT_MODELS:
+        return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
+
     # Check for duplicate submission
-    if out_path.split("eval-queue/")[1].lower() in requested_models:
+    if f"{model}_{revision}_{precision}" in requested_models:
         return styled_warning("This model has been already submitted.")
 
     with open(out_path, "w") as f:
@@ -238,7 +191,6 @@ def add_new_eval(
         path_or_fileobj=out_path,
         path_in_repo=out_path.split("eval-queue/")[1],
         repo_id=QUEUE_REPO,
-        token=H4_TOKEN,
         repo_type="dataset",
         commit_message=f"Add {model} to eval queue",
     )
@@ -246,16 +198,19 @@ def add_new_eval(
     # remove the local file
     os.remove(out_path)
 
-    return styled_message("Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list.")
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )
 
 
-def refresh():
-    leaderboard_df = get_leaderboard_df()
+# Basics
+def refresh() -> list[pd.DataFrame]:
+    leaderboard_df = get_leaderboard_df(eval_results, eval_results_private, COLS, BENCHMARK_COLS)
     (
         finished_eval_queue_df,
         running_eval_queue_df,
         pending_eval_queue_df,
-    ) = get_evaluation_queue_df()
+    ) = get_evaluation_queue_df(eval_queue, eval_queue_private, EVAL_REQUESTS_PATH, EVAL_COLS)
     return (
         leaderboard_df,
         finished_eval_queue_df,
@@ -264,47 +219,68 @@ def refresh():
     )
 
 
-def search_table(df, leaderboard_table, query):
-    if AutoEvalColumn.model_type.name in leaderboard_table.columns:
+def change_tab(query_param: str):
+    query_param = query_param.replace("'", '"')
+    query_param = json.loads(query_param)
+
+    if isinstance(query_param, dict) and "tab" in query_param and query_param["tab"] == "evaluation":
+        return gr.Tabs.update(selected=1)
+    else:
+        return gr.Tabs.update(selected=0)
+
+
+# Searching and filtering
+def search_table(df: pd.DataFrame, current_columns_df: pd.DataFrame, query: str) -> pd.DataFrame:
+    current_columns = current_columns_df.columns
+    if AutoEvalColumn.model_type.name in current_columns:
         filtered_df = df[
             (df[AutoEvalColumn.dummy.name].str.contains(query, case=False))
             | (df[AutoEvalColumn.model_type.name].str.contains(query, case=False))
-            ]
+        ]
     else:
         filtered_df = df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]
-    return filtered_df[leaderboard_table.columns]
+    return filtered_df[current_columns]
 
 
-def select_columns(df, columns):
-    always_here_cols = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name]
-    # We use COLS to maintain sorting 
-    filtered_df = df[always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]]
+def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
+    always_here_cols = [
+        AutoEvalColumn.model_type_symbol.name,
+        AutoEvalColumn.model.name,
+    ]
+    # We use COLS to maintain sorting
+    filtered_df = df[
+        always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]
+    ]
     return filtered_df
 
-#TODO allow this to filter by values of any columns
-def filter_items(df, leaderboard_table, query):
-    if query == "all":
-        return df[leaderboard_table.columns]
-    else:
-        query = query[0] #take only the emoji character
-    if AutoEvalColumn.model_type_symbol.name in leaderboard_table.columns:
-        filtered_df = df[(df[AutoEvalColumn.model_type_symbol.name] == query)]
-    else:
-        return leaderboard_table.columns
-    return filtered_df[leaderboard_table.columns]
-
-def change_tab(query_param):
-    query_param = query_param.replace("'", '"')
-    query_param = json.loads(query_param)
+NUMERIC_INTERVALS = {
+    "< 1.5B": (0, 1.5),
+    "~3B": (1.5, 5),
+    "~7B": (6, 11),
+    "~13B": (12, 15),
+    "~35B": (16, 55),
+    "60B+": (55, 10000),
+}
+
+def filter_models(
+    df: pd.DataFrame, current_columns_df: pd.DataFrame, type_query: list, size_query: list, show_deleted: bool
+) -> pd.DataFrame:
+    current_columns = current_columns_df.columns
+
+    # Show all models
+    if show_deleted:
+        filtered_df = df[current_columns]
+    else:  # Show only still on the hub models
+        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True][current_columns]
+
+    type_emoji = [t[0] for t in type_query]
+    filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
+
+    numeric_interval = [NUMERIC_INTERVALS[s] for s in size_query]
+    params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
+    filtered_df = filtered_df[params_column.between(numeric_interval[0][0], numeric_interval[-1][1])]
 
-    if (
-        isinstance(query_param, dict)
-        and "tab" in query_param
-        and query_param["tab"] == "evaluation"
-    ):
-        return gr.Tabs.update(selected=1)
-    else:
-        return gr.Tabs.update(selected=0)
+    return filtered_df
 
 
 demo = gr.Blocks(css=custom_css)
@@ -315,34 +291,83 @@ with demo:
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
             with gr.Row():
-                shown_columns = gr.CheckboxGroup(
-                    choices = [c for c in COLS if c not in [AutoEvalColumn.dummy.name, AutoEvalColumn.model.name, AutoEvalColumn.model_type_symbol.name]], 
-                    value = [c for c in COLS_LITE if c not in [AutoEvalColumn.dummy.name, AutoEvalColumn.model.name, AutoEvalColumn.model_type_symbol.name]],
-                    label="Select columns to show", 
-                    elem_id="column-select", 
-                    interactive=True,
-                )
+                with gr.Column():
+                    with gr.Row():
+                        shown_columns = gr.CheckboxGroup(
+                            choices=[
+                                c
+                                for c in COLS
+                                if c
+                                not in [
+                                    AutoEvalColumn.dummy.name,
+                                    AutoEvalColumn.model.name,
+                                    AutoEvalColumn.model_type_symbol.name,
+                                    AutoEvalColumn.still_on_hub.name,
+                                ]
+                            ],
+                            value=[
+                                c
+                                for c in COLS_LITE
+                                if c
+                                not in [
+                                    AutoEvalColumn.dummy.name,
+                                    AutoEvalColumn.model.name,
+                                    AutoEvalColumn.model_type_symbol.name,
+                                    AutoEvalColumn.still_on_hub.name,
+                                ]
+                            ],
+                            label="Select columns to show",
+                            elem_id="column-select",
+                            interactive=True,
+                        )
+                    with gr.Row():
+                        deleted_models_visibility = gr.Checkbox(
+                            value=True, label="Show gated/private/deleted models", interactive=True
+                        )
                 with gr.Column(min_width=320):
                     search_bar = gr.Textbox(
-                        placeholder="🔍 Search for your model and press ENTER...",
+                        placeholder="🔍 찾고자 하는 모델 명을 입력하세요",
                         show_label=False,
                         elem_id="search-bar",
                     )
-                    filter_columns = gr.Radio(
-                        label="⏚ Filter model types",
-                        choices = [
-                            "all", 
-                            ModelType.PT.to_str(),
-                            ModelType.FT.to_str(),
-                            ModelType.IFT.to_str(),
-                            ModelType.RL.to_str(), 
-                        ],
-                        value="all",
-                        elem_id="filter-columns"
-                    )
+                    with gr.Box(elem_id="box-filter"):
+                        filter_columns_type = gr.CheckboxGroup(
+                            label="Model types",
+                            choices=[
+                                ModelType.PT.to_str(),
+                                ModelType.FT.to_str(),
+                                ModelType.IFT.to_str(),
+                                ModelType.RL.to_str(),
+                            ],
+                            value=[
+                                ModelType.PT.to_str(),
+                                ModelType.FT.to_str(),
+                                ModelType.IFT.to_str(),
+                                ModelType.RL.to_str(),
+                            ],
+                            interactive=True,
+                            elem_id="filter-columns-type",
+                        )
+                        filter_columns_size = gr.CheckboxGroup(
+                            label="Model sizes",
+                            choices=list(NUMERIC_INTERVALS.keys()),
+                            value=list(NUMERIC_INTERVALS.keys()),
+                            interactive=True,
+                            elem_id="filter-columns-size",
+                        )
+
             leaderboard_table = gr.components.Dataframe(
-                value=leaderboard_df[[AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + shown_columns.value+ [AutoEvalColumn.dummy.name]],
-                headers=[AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + shown_columns.value + [AutoEvalColumn.dummy.name],
+                value=leaderboard_df[
+                    [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name]
+                    + shown_columns.value
+                    + [AutoEvalColumn.dummy.name]
+                ],
+                headers=[
+                    AutoEvalColumn.model_type_symbol.name,
+                    AutoEvalColumn.model.name,
+                ]
+                + shown_columns.value
+                + [AutoEvalColumn.dummy.name],
                 datatype=TYPES,
                 max_rows=None,
                 elem_id="leaderboard-table",
@@ -360,11 +385,55 @@ with demo:
             )
             search_bar.submit(
                 search_table,
-                [hidden_leaderboard_table_for_search, leaderboard_table, search_bar],
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    search_bar,
+                ],
+                leaderboard_table,
+            )
+            shown_columns.change(
+                select_columns,
+                [hidden_leaderboard_table_for_search, shown_columns],
+                leaderboard_table,
+                queue=False,
+            )
+            filter_columns_type.change(
+                filter_models,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    filter_columns_type,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                ],
+                leaderboard_table,
+                queue=False,
+            )
+            filter_columns_size.change(
+                filter_models,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    filter_columns_type,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                ],
                 leaderboard_table,
+                queue=False,
+            )
+            deleted_models_visibility.change(
+                filter_models,
+                [
+                    hidden_leaderboard_table_for_search,
+                    leaderboard_table,
+                    filter_columns_type,
+                    filter_columns_size,
+                    deleted_models_visibility,
+                ],
+                leaderboard_table,
+                queue=False,
             )
-            shown_columns.change(select_columns, [hidden_leaderboard_table_for_search, shown_columns], leaderboard_table)
-            filter_columns.change(filter_items, [hidden_leaderboard_table_for_search, leaderboard_table, filter_columns], leaderboard_table)
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
@@ -374,7 +443,10 @@ with demo:
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
                 with gr.Column():
-                    with gr.Accordion(f"✅ 평가 완료 ({len(finished_eval_queue_df)})", open=False):
+                    with gr.Accordion(
+                        f"✅ 평가 완료 ({len(finished_eval_queue_df)})",
+                        open=False,
+                    ):
                         with gr.Row():
                             finished_eval_table = gr.components.Dataframe(
                                 value=finished_eval_queue_df,
@@ -382,7 +454,10 @@ with demo:
                                 datatype=EVAL_TYPES,
                                 max_rows=5,
                             )
-                    with gr.Accordion(f"🔄 평가 진행 대기열 ({len(running_eval_queue_df)})", open=False):
+                    with gr.Accordion(
+                        f"🔄 평가 진행 대기열 ({len(running_eval_queue_df)})",
+                        open=False,
+                    ):
                         with gr.Row():
                             running_eval_table = gr.components.Dataframe(
                                 value=running_eval_queue_df,
@@ -391,7 +466,10 @@ with demo:
                                 max_rows=5,
                             )
 
-                    with gr.Accordion(f"⏳ 평가 대기 대기열 ({len(pending_eval_queue_df)})", open=False):
+                    with gr.Accordion(
+                        f"⏳ 평가 대기 대기열 ({len(pending_eval_queue_df)})",
+                        open=False,
+                    ):
                         with gr.Row():
                             pending_eval_table = gr.components.Dataframe(
                                 value=pending_eval_queue_df,
@@ -405,20 +483,16 @@ with demo:
             with gr.Row():
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(
-                        label="revision", placeholder="main"
-                    )
-                    private = gr.Checkbox(
-                        False, label="Private", visible=not IS_PUBLIC
-                    )
+                    revision_name_textbox = gr.Textbox(label="revision", placeholder="main")
+                    private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
                     model_type = gr.Dropdown(
-                        choices=[                         
+                        choices=[
                             ModelType.PT.to_str(" : "),
                             ModelType.FT.to_str(" : "),
                             ModelType.IFT.to_str(" : "),
-                            ModelType.RL.to_str(" : "), 
-                        ], 
-                        label="Model type", 
+                            ModelType.RL.to_str(" : "),
+                        ],
+                        label="Model type",
                         multiselect=False,
                         value=None,
                         interactive=True,
@@ -426,22 +500,26 @@ with demo:
 
                 with gr.Column():
                     precision = gr.Dropdown(
-                        choices=["float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)"], 
-                        label="Precision", 
+                        choices=[
+                            "float16",
+                            "bfloat16",
+                            "8bit (LLM.int8)",
+                            "4bit (QLoRA / FP4)",
+                            "GPTQ"
+                        ],
+                        label="Precision",
                         multiselect=False,
                         value="float16",
                         interactive=True,
                     )
                     weight_type = gr.Dropdown(
                         choices=["Original", "Delta", "Adapter"],
-                        label="Weights type", 
+                        label="Weights type",
                         multiselect=False,
                         value="Original",
                         interactive=True,
                     )
-                    base_model_name_textbox = gr.Textbox(
-                        label="Base model (for delta or adapter weights)"
-                    )
+                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
 
             submit_button = gr.Button("제출하고 평가받기")
             submission_result = gr.Markdown()
@@ -454,7 +532,7 @@ with demo:
                     precision,
                     private,
                     weight_type,
-                    model_type
+                    model_type,
                 ],
                 submission_result,
             )
@@ -470,6 +548,7 @@ with demo:
                     running_eval_table,
                     pending_eval_table,
                 ],
+                api_name='refresh'
             )
 
     with gr.Row():

models_backlinks.py

@@ -0,0 +1 @@
+models = ['upstage/Llama-2-70b-instruct-v2', 'upstage/Llama-2-70b-instruct', 'upstage/llama-65b-instruct', 'upstage/llama-65b-instruct', 'upstage/llama-30b-instruct-2048', 'upstage/llama-30b-instruct', 'baseline']

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -24,7 +24,7 @@ gradio_client==0.1.3
 h11==0.14.0
 httpcore==0.17.0
 httpx==0.24.0
-huggingface-hub==0.13.4
+huggingface-hub==0.16.4
 idna==3.4
 Jinja2==3.1.2
 jsonschema==4.17.3
@@ -59,7 +59,7 @@ sniffio==1.3.0
 starlette==0.26.1
 toolz==0.12.0
 tqdm==4.65.0
-transformers==4.28.1
+transformers==4.32.0
 typing_extensions==4.5.0
 tzdata==2023.3
 tzlocal==4.3

src/assets/css_html_js.py

@@ -1,11 +1,4 @@
 custom_css = """
-#changelog-text {
-    font-size: 16px !important;
-}
-
-#changelog-text h2 {
-    font-size: 18px !important;
-}
 
 .markdown-text {
     font-size: 16px !important;
@@ -75,6 +68,38 @@ table th:first-child {
 #scale-logo .download {
     display: none;
 }
+#filter_type{
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span{
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap{
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner{  
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input{
+    width: 1px
+}
+#filter-columns-type{
+    border:0;
+    padding:0.5;
+}
+#filter-columns-size{
+    border:0;
+    padding:0.5;
+}
+#box-filter > .form{
+    border: 0
+}
 """
 
 get_window_url_params = """

src/assets/hardcoded_evals.py

@@ -1,4 +1,4 @@
-from src.utils_display import AutoEvalColumn, model_hyperlink
+from src.display_models.utils import AutoEvalColumn, model_hyperlink
 
 gpt4_values = {
     AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt4"),
@@ -6,9 +6,9 @@ gpt4_values = {
     AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 84.3,
     AutoEvalColumn.arc.name: 96.3,
-    AutoEvalColumn.hellaswag.name:  95.3,
-    AutoEvalColumn.mmlu.name:  86.4,
-    AutoEvalColumn.truthfulqa.name:  59.0,
+    AutoEvalColumn.hellaswag.name: 95.3,
+    AutoEvalColumn.mmlu.name: 86.4,
+    AutoEvalColumn.truthfulqa.name: 59.0,
     AutoEvalColumn.dummy.name: "GPT-4",
     AutoEvalColumn.model_type.name: "",
 }
@@ -19,9 +19,9 @@ gpt35_values = {
     AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 71.9,
     AutoEvalColumn.arc.name: 85.2,
-    AutoEvalColumn.hellaswag.name:  85.5,
-    AutoEvalColumn.mmlu.name:  70.0,
-    AutoEvalColumn.truthfulqa.name:  47.0,
+    AutoEvalColumn.hellaswag.name: 85.5,
+    AutoEvalColumn.mmlu.name: 70.0,
+    AutoEvalColumn.truthfulqa.name: 47.0,
     AutoEvalColumn.dummy.name: "GPT-3.5",
     AutoEvalColumn.model_type.name: "",
 }
@@ -32,10 +32,9 @@ baseline = {
     AutoEvalColumn.precision.name: None,
     AutoEvalColumn.average.name: 25.0,
     AutoEvalColumn.arc.name: 25.0,
-    AutoEvalColumn.hellaswag.name:  25.0,
-    AutoEvalColumn.mmlu.name:  25.0,
-    AutoEvalColumn.truthfulqa.name:  25.0,
+    AutoEvalColumn.hellaswag.name: 25.0,
+    AutoEvalColumn.mmlu.name: 25.0,
+    AutoEvalColumn.truthfulqa.name: 25.0,
     AutoEvalColumn.dummy.name: "baseline",
     AutoEvalColumn.model_type.name: "",
 }
-

src/assets/text_content.py

@@ -1,60 +1,4 @@
-from ..auto_leaderboard.model_metadata_type import ModelType
-
-CHANGELOG_TEXT = f"""
-## [2023-06-19]
-- Added model type column
-- Hid revision and 8bit columns since all models are the same atm
-
-## [2023-06-16]
-- Refactored code base
-- Added new columns: number of parameters, hub likes, license
-
-## [2023-06-13] 
-- Adjust description for TruthfulQA
-
-## [2023-06-12] 
-- Add Human & GPT-4 Evaluations
-
-## [2023-06-05] 
-- Increase concurrent thread count to 40
-- Search models on ENTER
-
-## [2023-06-02] 
-- Add a typeahead search bar
-- Use webhooks to automatically spawn a new Space when someone opens a PR
-- Start recording `submitted_time` for eval requests
-- Limit AutoEvalColumn max-width
-
-## [2023-05-30] 
-- Add a citation button
-- Simplify Gradio layout
-
-## [2023-05-29] 
-- Auto-restart every hour for the latest results
-- Sync with the internal version (minor style changes)
-
-## [2023-05-24] 
-- Add a baseline that has 25.0 for all values
-- Add CHANGELOG
-
-## [2023-05-23] 
-- Fix a CSS issue that made the leaderboard hard to read in dark mode
-
-## [2023-05-22] 
-- Display a success/error message after submitting evaluation requests
-- Reject duplicate submission
-- Do not display results that have incomplete results 
-- Display different queues for jobs that are RUNNING, PENDING, FINISHED status 
-
-## [2023-05-15] 
-- Fix a typo: from "TruthQA" to "QA"
-
-## [2023-05-10] 
-- Fix a bug that prevented auto-refresh
-
-## [2023-05-10] 
-- Release the leaderboard to public
-"""
+from src.display_models.model_metadata_type import ModelType
 
 TITLE = """<h1 align="center" id="space-title">🚀 Open Ko-LLM Leaderboard</h1>"""
 
@@ -70,7 +14,7 @@ INTRODUCTION_TEXT = f"""
 
 LLM_BENCHMARKS_TEXT = f"""
 # Context
-뛰어난 LLM 모델들이 앞다투어 공개되고 있지만 이는 대부분 영어 중심의, 영어 문화권에 익숙한 모델입니다. 저희는 한국어 리더보드 🚀 Open Ko-LLM을 운영하여 한국어와 한국 문화의 특성을 반영한 모델을 평가하고자 합니다. 이를 통해 한국어 사용자들이 편리하게 리더보드를 이용하고 참여하여 한국의 연구 수준 향상에 기여할 수 있기를 바랍니다. 
+뛰어난 LLM 모델들이 앞다투어 공개되고 있지만 이는 대부분 영어 중심의, 영어 문화권에 익숙한 모델입니다. 저희는 한국어 리더보드 🚀 Open Ko-LLM을 운영하여 한국어와 한국 문화의 특성을 반영한 모델을 평가하고자 합니다. 이를 통해 한국어 사용자들이 편리하게 리더보드를 이용하고 참여하여 한국의 연구 수준 향상에 기여할 수 있기를 바랍니다. 
 
 ## Icons
 {ModelType.PT.to_str(" : ")} model
@@ -122,7 +66,7 @@ To get more information about quantization, see:
 """
 
 EVALUATION_QUEUE_TEXT = f"""
-# 🚀 Open-Ko LLM 리더보드의 평가 큐입니다.
+# 🚀 Open-Ko LLM 리더보드의 평가 큐입니다.
 이곳에 추가된 모델들은 곧 자동적으로 KT의 GPU 위에서 평가될 예정입니다!
 
 ## <모델 제출 전 확인하면 좋은 것들>

src/auto_leaderboard/model_metadata_type.py

@@ -1,597 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-import glob
-import json
-import os
-from typing import Dict, List
-
-from ..utils_display import AutoEvalColumn
-
-@dataclass
-class ModelInfo:
-    name: str
-    symbol: str # emoji
-
-
-class ModelType(Enum):
-    PT = ModelInfo(name="pretrained", symbol="🟢")
-    FT = ModelInfo(name="fine-tuned", symbol="🔶")
-    IFT = ModelInfo(name="instruction-tuned", symbol="⭕")
-    RL = ModelInfo(name="RL-tuned", symbol="🟦")
-    Unknown = ModelInfo(name="Unknown, add type to request file!", symbol="?")
-
-    def to_str(self, separator = " "):
-        return f"{self.value.symbol}{separator}{self.value.name}" 
-
-
-TYPE_METADATA: Dict[str, ModelType] = {
-    'notstoic/PygmalionCoT-7b': ModelType.IFT,
-    'aisquared/dlite-v1-355m': ModelType.IFT,
-    'aisquared/dlite-v1-1_5b': ModelType.IFT,
-    'aisquared/dlite-v1-774m': ModelType.IFT,
-    'aisquared/dlite-v1-124m': ModelType.IFT,
-    'aisquared/chopt-2_7b': ModelType.IFT,
-    'aisquared/dlite-v2-124m': ModelType.IFT,
-    'aisquared/dlite-v2-774m': ModelType.IFT,
-    'aisquared/dlite-v2-1_5b': ModelType.IFT,
-    'aisquared/chopt-1_3b': ModelType.IFT,
-    'aisquared/dlite-v2-355m': ModelType.IFT,
-    'augtoma/qCammel-13': ModelType.IFT,
-    'Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload': ModelType.IFT,
-    'Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload': ModelType.IFT,
-    'TheBloke/alpaca-lora-65B-HF': ModelType.FT,
-    'TheBloke/tulu-7B-fp16': ModelType.IFT,
-    'TheBloke/guanaco-7B-HF': ModelType.FT,
-    'TheBloke/koala-7B-HF': ModelType.FT,
-    'TheBloke/wizardLM-7B-HF': ModelType.IFT,
-    'TheBloke/airoboros-13B-HF': ModelType.IFT,
-    'TheBloke/koala-13B-HF': ModelType.FT,
-    'TheBloke/Wizard-Vicuna-7B-Uncensored-HF': ModelType.FT,
-    'TheBloke/dromedary-65b-lora-HF': ModelType.IFT,
-    'TheBloke/wizardLM-13B-1.0-fp16': ModelType.IFT,
-    'TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16': ModelType.FT,
-    'TheBloke/Wizard-Vicuna-30B-Uncensored-fp16': ModelType.FT,
-    'TheBloke/wizard-vicuna-13B-HF': ModelType.IFT,
-    'TheBloke/UltraLM-13B-fp16': ModelType.IFT,
-    'TheBloke/OpenAssistant-FT-7-Llama-30B-HF': ModelType.FT,
-    'TheBloke/vicuna-13B-1.1-HF': ModelType.IFT,
-    'TheBloke/guanaco-13B-HF': ModelType.FT,
-    'TheBloke/guanaco-65B-HF': ModelType.FT,
-    'TheBloke/airoboros-7b-gpt4-fp16': ModelType.IFT,
-    'TheBloke/llama-30b-supercot-SuperHOT-8K-fp16': ModelType.IFT,
-    'TheBloke/Llama-2-13B-fp16': ModelType.PT,
-    'TheBloke/llama-2-70b-Guanaco-QLoRA-fp16': ModelType.FT,
-    'TheBloke/landmark-attention-llama7b-fp16': ModelType.IFT,
-    'TheBloke/Planner-7B-fp16': ModelType.IFT,
-    'TheBloke/Wizard-Vicuna-13B-Uncensored-HF': ModelType.FT,
-    'TheBloke/gpt4-alpaca-lora-13B-HF': ModelType.IFT,
-    'TheBloke/gpt4-x-vicuna-13B-HF': ModelType.IFT,
-    'TheBloke/gpt4-alpaca-lora_mlp-65B-HF': ModelType.IFT,
-    'TheBloke/tulu-13B-fp16': ModelType.IFT,
-    'TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16': ModelType.IFT,
-    'TheBloke/Llama-2-70B-fp16': ModelType.IFT,
-    'TheBloke/WizardLM-30B-fp16': ModelType.IFT,
-    'TheBloke/robin-13B-v2-fp16': ModelType.FT,
-    'TheBloke/robin-33B-v2-fp16': ModelType.FT,
-    'TheBloke/Vicuna-13B-CoT-fp16': ModelType.IFT,
-    'TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16': ModelType.IFT,
-    'TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16': ModelType.FT,
-    'TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16': ModelType.IFT,
-    'TheBloke/GPlatty-30B-SuperHOT-8K-fp16': ModelType.FT,
-    'TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16': ModelType.IFT,
-    'TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16': ModelType.IFT,
-    'jphme/orca_mini_v2_ger_7b': ModelType.IFT,
-    'Ejafa/vicuna_7B_vanilla_1.1': ModelType.FT,
-    'kevinpro/Vicuna-13B-CoT': ModelType.IFT,
-    'AlekseyKorshuk/pygmalion-6b-vicuna-chatml': ModelType.FT,
-    'AlekseyKorshuk/chatml-pyg-v1': ModelType.FT,
-    'concedo/Vicuzard-30B-Uncensored': ModelType.FT,
-    'concedo/OPT-19M-ChatSalad': ModelType.FT,
-    'concedo/Pythia-70M-ChatSalad': ModelType.FT,
-    'digitous/13B-HyperMantis': ModelType.IFT,
-    'digitous/Adventien-GPTJ': ModelType.FT,
-    'digitous/Alpacino13b': ModelType.IFT,
-    'digitous/GPT-R': ModelType.IFT,
-    'digitous/Javelin-R': ModelType.IFT,
-    'digitous/Javalion-GPTJ': ModelType.IFT,
-    'digitous/Javalion-R': ModelType.IFT,
-    'digitous/Skegma-GPTJ': ModelType.FT,
-    'digitous/Alpacino30b': ModelType.IFT,
-    'digitous/Janin-GPTJ': ModelType.FT,
-    'digitous/Janin-R': ModelType.FT,
-    'digitous/Javelin-GPTJ': ModelType.FT,
-    'SaylorTwift/gpt2_test': ModelType.PT,
-    'anton-l/gpt-j-tiny-random': ModelType.FT,
-    'Andron00e/YetAnother_Open-Llama-3B-LoRA-OpenOrca': ModelType.FT,
-    'Lazycuber/pyg-instruct-wizardlm': ModelType.FT,
-    'Lazycuber/Janemalion-6B': ModelType.FT,
-    'IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1': ModelType.FT,
-    'IDEA-CCNL/Ziya-LLaMA-13B-v1': ModelType.IFT,
-    'dsvv-cair/alpaca-cleaned-llama-30b-bf16': ModelType.FT,
-    'gpt2-medium': ModelType.PT,
-    'camel-ai/CAMEL-13B-Combined-Data': ModelType.IFT,
-    'camel-ai/CAMEL-13B-Role-Playing-Data': ModelType.FT,
-    'camel-ai/CAMEL-33B-Combined-Data': ModelType.IFT,
-    'PygmalionAI/pygmalion-6b': ModelType.FT,
-    'PygmalionAI/metharme-1.3b': ModelType.IFT,
-    'PygmalionAI/pygmalion-1.3b': ModelType.FT,
-    'PygmalionAI/pygmalion-350m': ModelType.FT,
-    'PygmalionAI/pygmalion-2.7b': ModelType.FT,
-    'medalpaca/medalpaca-7b': ModelType.FT,
-    'lilloukas/Platypus-30B': ModelType.IFT,
-    'lilloukas/GPlatty-30B': ModelType.FT,
-    'mncai/chatdoctor': ModelType.FT,
-    'chaoyi-wu/MedLLaMA_13B': ModelType.FT,
-    'LoupGarou/WizardCoder-Guanaco-15B-V1.0': ModelType.IFT,
-    'LoupGarou/WizardCoder-Guanaco-15B-V1.1': ModelType.FT,
-    'hakurei/instruct-12b': ModelType.IFT,
-    'hakurei/lotus-12B': ModelType.FT,
-    'shibing624/chinese-llama-plus-13b-hf': ModelType.IFT,
-    'shibing624/chinese-alpaca-plus-7b-hf': ModelType.IFT,
-    'shibing624/chinese-alpaca-plus-13b-hf': ModelType.IFT,
-    'mosaicml/mpt-7b-instruct': ModelType.IFT,
-    'mosaicml/mpt-30b-chat': ModelType.IFT,
-    'mosaicml/mpt-7b-storywriter': ModelType.FT,
-    'mosaicml/mpt-30b-instruct': ModelType.IFT,
-    'mosaicml/mpt-7b-chat': ModelType.IFT,
-    'mosaicml/mpt-30b': ModelType.PT,
-    'Corianas/111m': ModelType.IFT,
-    'Corianas/Quokka_1.3b': ModelType.IFT,
-    'Corianas/256_5epoch': ModelType.FT,
-    'Corianas/Quokka_256m': ModelType.IFT,
-    'Corianas/Quokka_590m': ModelType.IFT,
-    'Corianas/gpt-j-6B-Dolly': ModelType.FT,
-    'Corianas/Quokka_2.7b': ModelType.IFT,
-    'cyberagent/open-calm-7b': ModelType.FT,
-    'Aspik101/Nous-Hermes-13b-pl-lora_unload': ModelType.IFT,
-    'THUDM/chatglm2-6b': ModelType.IFT,
-    'MetaIX/GPT4-X-Alpasta-30b': ModelType.IFT,
-    'NYTK/PULI-GPTrio': ModelType.PT,
-    'EleutherAI/pythia-1.3b': ModelType.PT,
-    'EleutherAI/pythia-2.8b-deduped': ModelType.PT,
-    'EleutherAI/gpt-neo-125m': ModelType.PT,
-    'EleutherAI/pythia-160m': ModelType.PT,
-    'EleutherAI/gpt-neo-2.7B': ModelType.PT,
-    'EleutherAI/pythia-1b-deduped': ModelType.PT,
-    'EleutherAI/pythia-6.7b': ModelType.PT,
-    'EleutherAI/pythia-70m-deduped': ModelType.PT,
-    'EleutherAI/gpt-neox-20b': ModelType.PT,
-    'EleutherAI/pythia-1.4b-deduped': ModelType.PT,
-    'EleutherAI/pythia-2.7b': ModelType.PT,
-    'EleutherAI/pythia-6.9b-deduped': ModelType.PT,
-    'EleutherAI/pythia-70m': ModelType.PT,
-    'EleutherAI/gpt-j-6b': ModelType.PT,
-    'EleutherAI/pythia-12b-deduped': ModelType.PT,
-    'EleutherAI/gpt-neo-1.3B': ModelType.PT,
-    'EleutherAI/pythia-410m-deduped': ModelType.PT,
-    'EleutherAI/pythia-160m-deduped': ModelType.PT,
-    'EleutherAI/polyglot-ko-12.8b': ModelType.PT,
-    'EleutherAI/pythia-12b': ModelType.PT,
-    'roneneldan/TinyStories-33M': ModelType.PT,
-    'roneneldan/TinyStories-28M': ModelType.PT,
-    'roneneldan/TinyStories-1M': ModelType.PT,
-    'roneneldan/TinyStories-8M': ModelType.PT,
-    'roneneldan/TinyStories-3M': ModelType.PT,
-    'jerryjalapeno/nart-100k-7b': ModelType.FT,
-    'lmsys/vicuna-13b-v1.3': ModelType.IFT,
-    'lmsys/vicuna-7b-v1.3': ModelType.IFT,
-    'lmsys/vicuna-13b-v1.1': ModelType.IFT,
-    'lmsys/vicuna-13b-delta-v1.1': ModelType.IFT,
-    'lmsys/vicuna-7b-delta-v1.1': ModelType.IFT,
-    'abhiramtirumala/DialoGPT-sarcastic-medium': ModelType.FT,
-    'haonan-li/bactrian-x-llama-13b-merged': ModelType.IFT,
-    'Gryphe/MythoLogic-13b': ModelType.IFT,
-    'Gryphe/MythoBoros-13b': ModelType.IFT,
-    'pillowtalks-ai/delta13b': ModelType.FT,
-    'wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard': ModelType.FT,
-    'bigscience/bloom-7b1': ModelType.PT,
-    'bigcode/tiny_starcoder_py': ModelType.PT,
-    'bigcode/starcoderplus': ModelType.FT,
-    'bigcode/gpt_bigcode-santacoder': ModelType.PT,
-    'bigcode/starcoder': ModelType.PT,
-    'Open-Orca/OpenOrca-Preview1-13B': ModelType.IFT,
-    'microsoft/DialoGPT-large': ModelType.FT,
-    'microsoft/DialoGPT-small': ModelType.FT,
-    'microsoft/DialoGPT-medium': ModelType.FT,
-    'microsoft/CodeGPT-small-py': ModelType.FT,
-    'Tincando/fiction_story_generator': ModelType.FT,
-    'Pirr/pythia-13b-deduped-green_devil': ModelType.FT,
-    'Aeala/GPT4-x-AlpacaDente2-30b': ModelType.FT,
-    'Aeala/GPT4-x-AlpacaDente-30b': ModelType.FT,
-    'Aeala/GPT4-x-Alpasta-13b': ModelType.FT,
-    'Aeala/VicUnlocked-alpaca-30b': ModelType.IFT,
-    'Tap-M/Luna-AI-Llama2-Uncensored': ModelType.FT,
-    'illuin/test-custom-llama': ModelType.FT,
-    'dvruette/oasst-llama-13b-2-epochs': ModelType.FT,
-    'dvruette/oasst-gpt-neox-20b-1000-steps': ModelType.FT,
-    'dvruette/llama-13b-pretrained-dropout': ModelType.PT,
-    'dvruette/llama-13b-pretrained': ModelType.PT,
-    'dvruette/llama-13b-pretrained-sft-epoch-1': ModelType.FT,
-    'dvruette/llama-13b-pretrained-sft-do2': ModelType.FT,
-    'dvruette/oasst-gpt-neox-20b-3000-steps': ModelType.FT,
-    'dvruette/oasst-pythia-12b-pretrained-sft': ModelType.FT,
-    'dvruette/oasst-pythia-6.9b-4000-steps': ModelType.FT,
-    'dvruette/gpt-neox-20b-full-precision': ModelType.FT,
-    'dvruette/oasst-llama-13b-1000-steps': ModelType.FT,
-    'openlm-research/open_llama_7b_700bt_preview': ModelType.PT,
-    'openlm-research/open_llama_7b': ModelType.PT,
-    'openlm-research/open_llama_7b_v2': ModelType.PT,
-    'openlm-research/open_llama_3b': ModelType.PT,
-    'openlm-research/open_llama_13b': ModelType.PT,
-    'openlm-research/open_llama_3b_v2': ModelType.PT,
-    'PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged': ModelType.IFT,
-    'GeorgiaTechResearchInstitute/galpaca-30b': ModelType.IFT,
-    'GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct': ModelType.IFT,
-    'databricks/dolly-v2-7b': ModelType.IFT,
-    'databricks/dolly-v2-3b': ModelType.IFT,
-    'databricks/dolly-v2-12b': ModelType.IFT,
-    'Rachneet/gpt2-xl-alpaca': ModelType.FT,
-    'Locutusque/gpt2-conversational-or-qa': ModelType.FT,
-    'psyche/kogpt': ModelType.FT,
-    'NbAiLab/nb-gpt-j-6B-alpaca': ModelType.IFT,
-    'Mikael110/llama-2-7b-guanaco-fp16': ModelType.FT,
-    'Mikael110/llama-2-13b-guanaco-fp16': ModelType.FT,
-    'Fredithefish/CrimsonPajama': ModelType.IFT,
-    'Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K': ModelType.FT,
-    'Fredithefish/ScarletPajama-3B-HF': ModelType.FT,
-    'Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4': ModelType.IFT,
-    'acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1': ModelType.IFT,
-    'eachadea/vicuna-13b-1.1': ModelType.FT,
-    'eachadea/vicuna-7b-1.1': ModelType.FT,
-    'eachadea/vicuna-13b': ModelType.FT,
-    'openaccess-ai-collective/wizard-mega-13b': ModelType.IFT,
-    'openaccess-ai-collective/manticore-13b': ModelType.IFT,
-    'openaccess-ai-collective/manticore-30b-chat-pyg-alpha': ModelType.IFT,
-    'openaccess-ai-collective/minotaur-13b': ModelType.IFT,
-    'openaccess-ai-collective/minotaur-13b-fixed': ModelType.IFT,
-    'openaccess-ai-collective/hippogriff-30b-chat': ModelType.IFT,
-    'openaccess-ai-collective/manticore-13b-chat-pyg': ModelType.IFT,
-    'pythainlp/wangchanglm-7.5B-sft-enth': ModelType.IFT,
-    'pythainlp/wangchanglm-7.5B-sft-en-sharded': ModelType.IFT,
-    'euclaise/gpt-neox-122m-minipile-digits': ModelType.FT,
-    'stabilityai/StableBeluga1-Delta': ModelType.IFT,
-    'stabilityai/stablelm-tuned-alpha-7b': ModelType.IFT,
-    'stabilityai/StableBeluga2': ModelType.IFT,
-    'stabilityai/StableBeluga-13B': ModelType.IFT,
-    'stabilityai/StableBeluga-7B': ModelType.IFT,
-    'stabilityai/stablelm-base-alpha-7b': ModelType.PT,
-    'stabilityai/stablelm-base-alpha-3b': ModelType.PT,
-    'stabilityai/stablelm-tuned-alpha-3b': ModelType.IFT,
-    'alibidaran/medical_transcription_generator': ModelType.FT,
-    'CalderaAI/30B-Lazarus': ModelType.IFT,
-    'CalderaAI/13B-BlueMethod': ModelType.IFT,
-    'CalderaAI/13B-Ouroboros': ModelType.IFT,
-    'KoboldAI/OPT-13B-Erebus': ModelType.FT,
-    'KoboldAI/GPT-J-6B-Janeway': ModelType.FT,
-    'KoboldAI/GPT-J-6B-Shinen': ModelType.FT,
-    'KoboldAI/fairseq-dense-2.7B': ModelType.PT,
-    'KoboldAI/OPT-6B-nerys-v2': ModelType.FT,
-    'KoboldAI/GPT-NeoX-20B-Skein': ModelType.FT,
-    'KoboldAI/PPO_Pygway-6b-Mix': ModelType.FT,
-    'KoboldAI/fairseq-dense-6.7B': ModelType.PT,
-    'KoboldAI/fairseq-dense-125M': ModelType.PT,
-    'KoboldAI/OPT-13B-Nerybus-Mix': ModelType.FT,
-    'KoboldAI/OPT-2.7B-Erebus': ModelType.FT,
-    'KoboldAI/OPT-350M-Nerys-v2': ModelType.FT,
-    'KoboldAI/OPT-2.7B-Nerys-v2': ModelType.FT,
-    'KoboldAI/OPT-2.7B-Nerybus-Mix': ModelType.FT,
-    'KoboldAI/OPT-13B-Nerys-v2': ModelType.FT,
-    'KoboldAI/GPT-NeoX-20B-Erebus': ModelType.FT,
-    'KoboldAI/OPT-6.7B-Erebus': ModelType.FT,
-    'KoboldAI/fairseq-dense-355M': ModelType.PT,
-    'KoboldAI/OPT-6.7B-Nerybus-Mix': ModelType.FT,
-    'KoboldAI/GPT-J-6B-Adventure': ModelType.FT,
-    'KoboldAI/OPT-350M-Erebus': ModelType.FT,
-    'KoboldAI/GPT-J-6B-Skein': ModelType.FT,
-    'KoboldAI/OPT-30B-Erebus': ModelType.FT,
-    'klosax/pythia-160m-deduped-step92k-193bt': ModelType.PT,
-    'klosax/open_llama_3b_350bt_preview': ModelType.PT,
-    'klosax/openllama-3b-350bt': ModelType.PT,
-    'klosax/pythia-70m-deduped-step44k-92bt': ModelType.PT,
-    'klosax/open_llama_13b_600bt_preview': ModelType.PT,
-    'klosax/open_llama_7b_400bt_preview': ModelType.PT,
-    'kfkas/Llama-2-ko-7b-Chat': ModelType.IFT,
-    'WeOpenML/Alpaca-7B-v1': ModelType.IFT,
-    'WeOpenML/PandaLM-Alpaca-7B-v1': ModelType.IFT,
-    'TFLai/gpt2-turkish-uncased': ModelType.FT,
-    'ehartford/WizardLM-13B-Uncensored': ModelType.IFT,
-    'ehartford/dolphin-llama-13b': ModelType.IFT,
-    'ehartford/Wizard-Vicuna-30B-Uncensored': ModelType.FT,
-    'ehartford/WizardLM-30B-Uncensored': ModelType.IFT,
-    'ehartford/Wizard-Vicuna-13B-Uncensored': ModelType.FT,
-    'ehartford/WizardLM-7B-Uncensored': ModelType.IFT,
-    'ehartford/based-30b': ModelType.FT,
-    'ehartford/Wizard-Vicuna-7B-Uncensored': ModelType.FT,
-    'wahaha1987/llama_7b_sharegpt94k_fastchat': ModelType.FT,
-    'wahaha1987/llama_13b_sharegpt94k_fastchat': ModelType.FT,
-    'OpenAssistant/oasst-sft-1-pythia-12b': ModelType.FT,
-    'OpenAssistant/stablelm-7b-sft-v7-epoch-3': ModelType.IFT,
-    'OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5': ModelType.FT,
-    'OpenAssistant/pythia-12b-sft-v8-2.5k-steps': ModelType.IFT,
-    'OpenAssistant/pythia-12b-sft-v8-7k-steps': ModelType.IFT,
-    'OpenAssistant/pythia-12b-pre-v8-12.5k-steps': ModelType.IFT,
-    'OpenAssistant/llama2-13b-orca-8k-3319': ModelType.IFT,
-    'junelee/wizard-vicuna-13b': ModelType.FT,
-    'BreadAi/gpt-YA-1-1_160M': ModelType.PT,
-    'BreadAi/MuseCan': ModelType.PT,
-    'BreadAi/MusePy-1-2': ModelType.PT,
-    'BreadAi/DiscordPy': ModelType.PT,
-    'BreadAi/PM_modelV2': ModelType.PT,
-    'BreadAi/gpt-Youtube': ModelType.PT,
-    'BreadAi/StoryPy': ModelType.FT,
-    'julianweng/Llama-2-7b-chat-orcah': ModelType.FT,
-    'AGI-inc/lora_moe_7b_baseline': ModelType.FT,
-    'AGI-inc/lora_moe_7b': ModelType.FT,
-    'togethercomputer/GPT-NeoXT-Chat-Base-20B': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Chat-7B-v0.1': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-7B-Base': ModelType.PT,
-    'togethercomputer/RedPajama-INCITE-7B-Instruct': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Base-3B-v1': ModelType.PT,
-    'togethercomputer/Pythia-Chat-Base-7B': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Base-7B-v0.1': ModelType.PT,
-    'togethercomputer/GPT-JT-6B-v1': ModelType.IFT,
-    'togethercomputer/GPT-JT-6B-v0': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Chat-3B-v1': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-7B-Chat': ModelType.IFT,
-    'togethercomputer/RedPajama-INCITE-Instruct-3B-v1': ModelType.IFT,
-    'Writer/camel-5b-hf': ModelType.IFT,
-    'Writer/palmyra-base': ModelType.PT,
-    'MBZUAI/LaMini-GPT-1.5B': ModelType.IFT,
-    'MBZUAI/lamini-cerebras-111m': ModelType.IFT,
-    'MBZUAI/lamini-neo-1.3b': ModelType.IFT,
-    'MBZUAI/lamini-cerebras-1.3b': ModelType.IFT,
-    'MBZUAI/lamini-cerebras-256m': ModelType.IFT,
-    'MBZUAI/LaMini-GPT-124M': ModelType.IFT,
-    'MBZUAI/lamini-neo-125m': ModelType.IFT,
-    'TehVenom/DiffMerge-DollyGPT-Pygmalion': ModelType.FT,
-    'TehVenom/PPO_Shygmalion-6b': ModelType.FT,
-    'TehVenom/Dolly_Shygmalion-6b-Dev_V8P2': ModelType.FT,
-    'TehVenom/Pygmalion_AlpacaLora-7b': ModelType.FT,
-    'TehVenom/PPO_Pygway-V8p4_Dev-6b': ModelType.FT,
-    'TehVenom/Dolly_Malion-6b': ModelType.FT,
-    'TehVenom/PPO_Shygmalion-V8p4_Dev-6b': ModelType.FT,
-    'TehVenom/ChanMalion': ModelType.FT,
-    'TehVenom/GPT-J-Pyg_PPO-6B': ModelType.IFT,
-    'TehVenom/Pygmalion-13b-Merged': ModelType.FT,
-    'TehVenom/Metharme-13b-Merged': ModelType.IFT,
-    'TehVenom/Dolly_Shygmalion-6b': ModelType.FT,
-    'TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4': ModelType.IFT,
-    'georgesung/llama2_7b_chat_uncensored': ModelType.FT,
-    'vicgalle/gpt2-alpaca': ModelType.IFT,
-    'vicgalle/alpaca-7b': ModelType.FT,
-    'vicgalle/gpt2-alpaca-gpt4': ModelType.IFT,
-    'facebook/opt-350m': ModelType.PT,
-    'facebook/opt-125m': ModelType.PT,
-    'facebook/xglm-4.5B': ModelType.PT,
-    'facebook/opt-2.7b': ModelType.PT,
-    'facebook/opt-6.7b': ModelType.PT,
-    'facebook/galactica-30b': ModelType.PT,
-    'facebook/opt-13b': ModelType.PT,
-    'facebook/opt-66b': ModelType.PT,
-    'facebook/xglm-7.5B': ModelType.PT,
-    'facebook/xglm-564M': ModelType.PT,
-    'facebook/opt-30b': ModelType.PT,
-    'golaxy/gogpt-7b': ModelType.FT,
-    'golaxy/gogpt2-7b': ModelType.FT,
-    'golaxy/gogpt-7b-bloom': ModelType.FT,
-    'golaxy/gogpt-3b-bloom': ModelType.FT,
-    'psmathur/orca_mini_v2_7b': ModelType.IFT,
-    'psmathur/orca_mini_7b': ModelType.IFT,
-    'psmathur/orca_mini_3b': ModelType.IFT,
-    'psmathur/orca_mini_v2_13b': ModelType.IFT,
-    'gpt2-xl': ModelType.PT,
-    'lxe/Cerebras-GPT-2.7B-Alpaca-SP': ModelType.FT,
-    'Monero/Manticore-13b-Chat-Pyg-Guanaco': ModelType.FT,
-    'Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b': ModelType.IFT,
-    'Monero/WizardLM-13b-OpenAssistant-Uncensored': ModelType.IFT,
-    'Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b': ModelType.IFT,
-    'jzjiao/opt-1.3b-rlhf': ModelType.FT,
-    'HuggingFaceH4/starchat-beta': ModelType.IFT,
-    'KnutJaegersberg/gpt-2-xl-EvolInstruct': ModelType.IFT,
-    'KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct': ModelType.IFT,
-    'KnutJaegersberg/galactica-orca-wizardlm-1.3b': ModelType.IFT,
-    'openchat/openchat_8192': ModelType.IFT,
-    'openchat/openchat_v2': ModelType.IFT,
-    'openchat/openchat_v2_w': ModelType.IFT,
-    'ausboss/llama-13b-supercot': ModelType.IFT,
-    'ausboss/llama-30b-supercot': ModelType.IFT,
-    'Neko-Institute-of-Science/metharme-7b': ModelType.IFT,
-    'Neko-Institute-of-Science/pygmalion-7b': ModelType.FT,
-    'SebastianSchramm/Cerebras-GPT-111M-instruction': ModelType.IFT,
-    'victor123/WizardLM-13B-1.0': ModelType.IFT,
-    'OpenBuddy/openbuddy-openllama-13b-v7-fp16': ModelType.FT,
-    'OpenBuddy/openbuddy-llama2-13b-v8.1-fp16': ModelType.FT,
-    'OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16': ModelType.FT,
-    'baichuan-inc/Baichuan-7B': ModelType.PT,
-    'tiiuae/falcon-40b-instruct': ModelType.IFT,
-    'tiiuae/falcon-40b': ModelType.PT,
-    'tiiuae/falcon-7b': ModelType.PT,
-    'YeungNLP/firefly-llama-13b': ModelType.FT,
-    'YeungNLP/firefly-llama-13b-v1.2': ModelType.FT,
-    'YeungNLP/firefly-llama2-13b': ModelType.FT,
-    'YeungNLP/firefly-ziya-13b': ModelType.FT,
-    'shaohang/Sparse0.5_OPT-1.3': ModelType.FT,
-    'xzuyn/Alpacino-SuperCOT-13B': ModelType.IFT,
-    'xzuyn/MedicWizard-7B': ModelType.FT,
-    'xDAN-AI/xDAN_13b_l2_lora': ModelType.FT,
-    'beomi/KoAlpaca-Polyglot-5.8B': ModelType.FT,
-    'beomi/llama-2-ko-7b': ModelType.IFT,
-    'Salesforce/codegen-6B-multi': ModelType.PT,
-    'Salesforce/codegen-16B-nl': ModelType.PT,
-    'Salesforce/codegen-6B-nl': ModelType.PT,
-    'ai-forever/rugpt3large_based_on_gpt2': ModelType.FT,
-    'gpt2-large': ModelType.PT,
-    'frank098/orca_mini_3b_juniper': ModelType.FT,
-    'frank098/WizardLM_13B_juniper': ModelType.FT,
-    'FPHam/Free_Sydney_13b_HF': ModelType.FT,
-    'huggingface/llama-13b': ModelType.PT,
-    'huggingface/llama-7b': ModelType.PT,
-    'huggingface/llama-65b': ModelType.PT,
-    'huggingface/llama-30b': ModelType.PT,
-    'Henk717/chronoboros-33B': ModelType.IFT,
-    'jondurbin/airoboros-13b-gpt4-1.4': ModelType.IFT,
-    'jondurbin/airoboros-7b': ModelType.IFT,
-    'jondurbin/airoboros-7b-gpt4': ModelType.IFT,
-    'jondurbin/airoboros-7b-gpt4-1.1': ModelType.IFT,
-    'jondurbin/airoboros-7b-gpt4-1.2': ModelType.IFT,
-    'jondurbin/airoboros-7b-gpt4-1.3': ModelType.IFT,
-    'jondurbin/airoboros-7b-gpt4-1.4': ModelType.IFT,
-    'jondurbin/airoboros-l2-7b-gpt4-1.4.1': ModelType.IFT,
-    'jondurbin/airoboros-l2-13b-gpt4-1.4.1': ModelType.IFT,
-    'jondurbin/airoboros-l2-70b-gpt4-1.4.1': ModelType.IFT,
-    'jondurbin/airoboros-13b': ModelType.IFT,
-    'jondurbin/airoboros-33b-gpt4-1.4': ModelType.IFT,
-    'jondurbin/airoboros-33b-gpt4-1.2': ModelType.IFT,
-    'jondurbin/airoboros-65b-gpt4-1.2': ModelType.IFT,
-    'ariellee/SuperPlatty-30B': ModelType.IFT,
-    'danielhanchen/open_llama_3b_600bt_preview': ModelType.FT,
-    'cerebras/Cerebras-GPT-256M': ModelType.PT,
-    'cerebras/Cerebras-GPT-1.3B': ModelType.PT,
-    'cerebras/Cerebras-GPT-13B': ModelType.PT,
-    'cerebras/Cerebras-GPT-2.7B': ModelType.PT,
-    'cerebras/Cerebras-GPT-111M': ModelType.PT,
-    'cerebras/Cerebras-GPT-6.7B': ModelType.PT,
-    'Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf': ModelType.RL,
-    'Yhyu13/llama-30B-hf-openassitant': ModelType.FT,
-    'NousResearch/Nous-Hermes-Llama2-13b': ModelType.IFT,
-    'NousResearch/Nous-Hermes-llama-2-7b': ModelType.IFT,
-    'NousResearch/Redmond-Puffin-13B': ModelType.IFT,
-    'NousResearch/Nous-Hermes-13b': ModelType.IFT,
-    'project-baize/baize-v2-7b': ModelType.IFT,
-    'project-baize/baize-v2-13b': ModelType.IFT,
-    'LLMs/WizardLM-13B-V1.0': ModelType.FT,
-    'LLMs/AlpacaGPT4-7B-elina': ModelType.FT,
-    'wenge-research/yayi-7b': ModelType.FT,
-    'wenge-research/yayi-7b-llama2': ModelType.FT,
-    'wenge-research/yayi-13b-llama2': ModelType.FT,
-    'yhyhy3/open_llama_7b_v2_med_instruct': ModelType.IFT,
-    'llama-anon/instruct-13b': ModelType.IFT,
-    'huggingtweets/jerma985': ModelType.FT,
-    'huggingtweets/gladosystem': ModelType.FT,
-    'huggingtweets/bladeecity-jerma985': ModelType.FT,
-    'huggyllama/llama-13b': ModelType.PT,
-    'huggyllama/llama-65b': ModelType.PT,
-    'FabbriSimo01/Facebook_opt_1.3b_Quantized': ModelType.PT,
-    'upstage/Llama-2-70b-instruct': ModelType.IFT,
-    'upstage/Llama-2-70b-instruct-1024': ModelType.IFT,
-    'upstage/llama-65b-instruct': ModelType.IFT,
-    'upstage/llama-30b-instruct-2048': ModelType.IFT,
-    'upstage/llama-30b-instruct': ModelType.IFT,
-    'WizardLM/WizardLM-13B-1.0': ModelType.IFT,
-    'WizardLM/WizardLM-13B-V1.1': ModelType.IFT,
-    'WizardLM/WizardLM-13B-V1.2': ModelType.IFT,
-    'WizardLM/WizardLM-30B-V1.0': ModelType.IFT,
-    'WizardLM/WizardCoder-15B-V1.0': ModelType.IFT,
-    'gpt2': ModelType.PT,
-    'keyfan/vicuna-chinese-replication-v1.1': ModelType.IFT,
-    'nthngdy/pythia-owt2-70m-100k': ModelType.FT,
-    'nthngdy/pythia-owt2-70m-50k': ModelType.FT,
-    'quantumaikr/KoreanLM-hf': ModelType.FT,
-    'quantumaikr/open_llama_7b_hf': ModelType.FT,
-    'quantumaikr/QuantumLM-70B-hf': ModelType.IFT,
-    'MayaPH/FinOPT-Lincoln': ModelType.FT,
-    'MayaPH/FinOPT-Franklin': ModelType.FT,
-    'MayaPH/GodziLLa-30B': ModelType.IFT,
-    'MayaPH/GodziLLa-30B-plus': ModelType.IFT,
-    'MayaPH/FinOPT-Washington': ModelType.FT,
-    'ogimgio/gpt-neo-125m-neurallinguisticpioneers': ModelType.FT,
-    'layoric/llama-2-13b-code-alpaca': ModelType.FT,
-    'CobraMamba/mamba-gpt-3b': ModelType.FT,
-    'CobraMamba/mamba-gpt-3b-v2': ModelType.FT,
-    'CobraMamba/mamba-gpt-3b-v3': ModelType.FT,
-    'timdettmers/guanaco-33b-merged': ModelType.FT,
-    'elinas/chronos-33b': ModelType.IFT,
-    'heegyu/RedTulu-Uncensored-3B-0719': ModelType.IFT,
-    'heegyu/WizardVicuna-Uncensored-3B-0719': ModelType.IFT,
-    'heegyu/WizardVicuna-3B-0719': ModelType.IFT,
-    'meta-llama/Llama-2-7b-chat-hf': ModelType.RL,
-    'meta-llama/Llama-2-7b-hf': ModelType.PT,
-    'meta-llama/Llama-2-13b-chat-hf': ModelType.RL,
-    'meta-llama/Llama-2-13b-hf': ModelType.PT,
-    'meta-llama/Llama-2-70b-chat-hf': ModelType.RL,
-    'meta-llama/Llama-2-70b-hf': ModelType.PT,
-    'xhyi/PT_GPTNEO350_ATG': ModelType.FT,
-    'h2oai/h2ogpt-gm-oasst1-en-1024-20b': ModelType.FT,
-    'h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt': ModelType.FT,
-    'h2oai/h2ogpt-oig-oasst1-512-6_9b': ModelType.IFT,
-    'h2oai/h2ogpt-oasst1-512-12b': ModelType.IFT,
-    'h2oai/h2ogpt-oig-oasst1-256-6_9b': ModelType.IFT,
-    'h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt': ModelType.FT,
-    'h2oai/h2ogpt-oasst1-512-20b': ModelType.IFT,
-    'h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2': ModelType.FT,
-    'h2oai/h2ogpt-gm-oasst1-en-1024-12b': ModelType.FT,
-    'h2oai/h2ogpt-gm-oasst1-multilang-1024-20b': ModelType.FT,
-    'bofenghuang/vigogne-13b-instruct': ModelType.IFT,
-    'bofenghuang/vigogne-13b-chat': ModelType.FT,
-    'bofenghuang/vigogne-2-7b-instruct': ModelType.IFT,
-    'bofenghuang/vigogne-7b-instruct': ModelType.IFT,
-    'bofenghuang/vigogne-7b-chat': ModelType.FT,
-    'Vmware/open-llama-7b-v2-open-instruct': ModelType.IFT,
-    'VMware/open-llama-0.7T-7B-open-instruct-v1.1': ModelType.IFT,
-    'ewof/koishi-instruct-3b': ModelType.IFT,
-    'gywy/llama2-13b-chinese-v1': ModelType.FT,
-    'GOAT-AI/GOAT-7B-Community': ModelType.FT,
-    'psyche/kollama2-7b': ModelType.FT,
-    'TheTravellingEngineer/llama2-7b-hf-guanaco': ModelType.FT,
-    'beaugogh/pythia-1.4b-deduped-sharegpt': ModelType.FT,
-    'augtoma/qCammel-70-x': ModelType.IFT,
-    'Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload': ModelType.IFT,
-    'anhnv125/pygmalion-6b-roleplay': ModelType.FT,
-    '64bits/LexPodLM-13B': ModelType.FT,
-}
-
-
-def model_type_from_str(type):
-    if "fine-tuned" in type or "🔶" in type:
-        return ModelType.FT
-    if "pretrained" in type or "🟢" in type:
-        return ModelType.PT
-    if "RL-tuned" in type or "🟦" in type:
-        return ModelType.RL
-    if "instruction-tuned" in type or "⭕" in type:
-        return ModelType.IFT
-    return ModelType.Unknown
-
-
-def get_model_type(leaderboard_data: List[dict]):
-    for model_data in leaderboard_data:
-        request_files = os.path.join("eval-queue", model_data["model_name_for_query"] + "_eval_request_*" + ".json")
-        request_files = glob.glob(request_files)
-
-        request_file = ""
-        if len(request_files) == 1:
-            request_file = request_files[0]
-        elif len(request_files) > 1:
-            request_files = sorted(request_files, reverse=True)
-            for tmp_request_file in request_files:
-                with open(tmp_request_file, "r") as f:
-                    req_content = json.load(f)
-                    if req_content["status"] == "FINISHED" and req_content["precision"] == model_data["Precision"].split(".")[-1]: 
-                        request_file = tmp_request_file
-        
-        if request_file == "":
-            model_data[AutoEvalColumn.model_type.name] = ""
-            model_data[AutoEvalColumn.model_type_symbol.name] = ""
-            continue
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            is_delta = request["weight_type"] != "Original"
-        except Exception:
-            is_delta = False
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            model_type = model_type_from_str(request["model_type"])
-            model_data[AutoEvalColumn.model_type.name] = model_type.value.name
-            model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol + ("🔺" if is_delta else "")
-        except KeyError:
-            if model_data["model_name_for_query"] in TYPE_METADATA:
-                model_data[AutoEvalColumn.model_type.name] = TYPE_METADATA[model_data["model_name_for_query"]].value.name
-                model_data[AutoEvalColumn.model_type_symbol.name] = TYPE_METADATA[model_data["model_name_for_query"]].value.symbol + ("🔺" if is_delta else "")
-            else:
-                model_data[AutoEvalColumn.model_type.name] = ModelType.Unknown.value.name
-                model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol

src/{auto_leaderboard → display_models}/get_model_metadata.py

@@ -1,17 +1,22 @@
-import re
+import glob
+import json
 import os
+import re
 from typing import List
 
-from src.utils_display import AutoEvalColumn
-from src.auto_leaderboard.model_metadata_type import get_model_type
-
-from huggingface_hub import HfApi
 import huggingface_hub
+from huggingface_hub import HfApi
+from tqdm import tqdm
+
+from src.display_models.model_metadata_flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
+from src.display_models.model_metadata_type import MODEL_TYPE_METADATA, ModelType, model_type_from_str
+from src.display_models.utils import AutoEvalColumn, model_hyperlink
+
 api = HfApi(token=os.environ.get("H4_TOKEN", None))
 
 
 def get_model_infos_from_hub(leaderboard_data: List[dict]):
-    for model_data in leaderboard_data:
+    for model_data in tqdm(leaderboard_data):
         model_name = model_data["model_name_for_query"]
         try:
             model_info = api.model_info(model_name)
@@ -33,15 +38,18 @@ def get_model_license(model_info):
     except Exception:
         return None
 
+
 def get_model_likes(model_info):
     return model_info.likes
 
+
 size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
 
+
 def get_model_size(model_name, model_info):
     # In billions
     try:
-        return round(model_info.safetensors["total"] / 1e9, 3) 
+        return round(model_info.safetensors["total"] / 1e9, 3)
     except AttributeError:
         try:
             size_match = re.search(size_pattern, model_name.lower())
@@ -51,6 +59,74 @@ def get_model_size(model_name, model_info):
             return None
 
 
+def get_model_type(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        request_files = os.path.join(
+            "eval-queue",
+            model_data["model_name_for_query"] + "_eval_request_*" + ".json",
+        )
+        request_files = glob.glob(request_files)
+
+        # Select correct request file (precision)
+        request_file = ""
+        if len(request_files) == 1:
+            request_file = request_files[0]
+        elif len(request_files) > 1:
+            request_files = sorted(request_files, reverse=True)
+            for tmp_request_file in request_files:
+                with open(tmp_request_file, "r") as f:
+                    req_content = json.load(f)
+                    if (
+                        req_content["status"] == "FINISHED"
+                        and req_content["precision"] == model_data["Precision"].split(".")[-1]
+                    ):
+                        request_file = tmp_request_file
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            model_type = model_type_from_str(request["model_type"])
+            model_data[AutoEvalColumn.model_type.name] = model_type.value.name
+            model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol  # + ("🔺" if is_delta else "")
+        except Exception:
+            if model_data["model_name_for_query"] in MODEL_TYPE_METADATA:
+                model_data[AutoEvalColumn.model_type.name] = MODEL_TYPE_METADATA[
+                    model_data["model_name_for_query"]
+                ].value.name
+                model_data[AutoEvalColumn.model_type_symbol.name] = MODEL_TYPE_METADATA[
+                    model_data["model_name_for_query"]
+                ].value.symbol  # + ("🔺" if is_delta else "")
+            else:
+                model_data[AutoEvalColumn.model_type.name] = ModelType.Unknown.value.name
+                model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol
+
+
+def flag_models(leaderboard_data: List[dict]):
+    for model_data in leaderboard_data:
+        if model_data["model_name_for_query"] in FLAGGED_MODELS:
+            issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
+            issue_link = model_hyperlink(
+                FLAGGED_MODELS[model_data["model_name_for_query"]],
+                f"See discussion #{issue_num}",
+            )
+            model_data[
+                AutoEvalColumn.model.name
+            ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
+
+
+def remove_forbidden_models(leaderboard_data: List[dict]):
+    indices_to_remove = []
+    for ix, model in enumerate(leaderboard_data):
+        if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
+            indices_to_remove.append(ix)
+
+    for ix in reversed(indices_to_remove):
+        leaderboard_data.pop(ix)
+    return leaderboard_data
+
+
 def apply_metadata(leaderboard_data: List[dict]):
+    leaderboard_data = remove_forbidden_models(leaderboard_data)
     get_model_type(leaderboard_data)
     get_model_infos_from_hub(leaderboard_data)
+    flag_models(leaderboard_data)

src/display_models/model_metadata_flags.py

@@ -0,0 +1,15 @@
+# Models which have been flagged by users as being problematic for a reason or another
+# (Model name to forum discussion link)
+FLAGGED_MODELS = {
+    "Voicelab/trurl-2-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/202",
+    "deepnight-research/llama-2-70B-inst": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/207",
+    "Aspik101/trurl-2-13b-pl-instruct_unload": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/213",
+    "Fredithefish/ReasonixPajama-3B-HF": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/236",
+    "TigerResearch/tigerbot-7b-sft-v1": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/237",
+    "gaodrew/gaodrew-gorgonzola-13b": "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/discussions/215",
+}
+
+# Models which have been requested by orgs to not be submitted on the leaderboard
+DO_NOT_SUBMIT_MODELS = [
+    "Voicelab/trurl-2-13b",  # trained on MMLU
+]

src/display_models/model_metadata_type.py

@@ -0,0 +1,553 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict
+
+
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str  # emoji
+
+
+class ModelType(Enum):
+    PT = ModelInfo(name="pretrained", symbol="🟢")
+    FT = ModelInfo(name="fine-tuned", symbol="🔶")
+    IFT = ModelInfo(name="instruction-tuned", symbol="⭕")
+    RL = ModelInfo(name="RL-tuned", symbol="🟦")
+    Unknown = ModelInfo(name="Unknown, add type to request file!", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+
+MODEL_TYPE_METADATA: Dict[str, ModelType] = {
+    "tiiuae/falcon-180B": ModelType.PT,
+    "Qwen/Qwen-7B": ModelType.PT,
+    "Qwen/Qwen-7B-Chat": ModelType.RL,
+    "notstoic/PygmalionCoT-7b": ModelType.IFT,
+    "aisquared/dlite-v1-355m": ModelType.IFT,
+    "aisquared/dlite-v1-1_5b": ModelType.IFT,
+    "aisquared/dlite-v1-774m": ModelType.IFT,
+    "aisquared/dlite-v1-124m": ModelType.IFT,
+    "aisquared/chopt-2_7b": ModelType.IFT,
+    "aisquared/dlite-v2-124m": ModelType.IFT,
+    "aisquared/dlite-v2-774m": ModelType.IFT,
+    "aisquared/dlite-v2-1_5b": ModelType.IFT,
+    "aisquared/chopt-1_3b": ModelType.IFT,
+    "aisquared/dlite-v2-355m": ModelType.IFT,
+    "augtoma/qCammel-13": ModelType.IFT,
+    "Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload": ModelType.IFT,
+    "Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload": ModelType.IFT,
+    "TheBloke/alpaca-lora-65B-HF": ModelType.FT,
+    "TheBloke/tulu-7B-fp16": ModelType.IFT,
+    "TheBloke/guanaco-7B-HF": ModelType.FT,
+    "TheBloke/koala-7B-HF": ModelType.FT,
+    "TheBloke/wizardLM-7B-HF": ModelType.IFT,
+    "TheBloke/airoboros-13B-HF": ModelType.IFT,
+    "TheBloke/koala-13B-HF": ModelType.FT,
+    "TheBloke/Wizard-Vicuna-7B-Uncensored-HF": ModelType.FT,
+    "TheBloke/dromedary-65b-lora-HF": ModelType.IFT,
+    "TheBloke/wizardLM-13B-1.0-fp16": ModelType.IFT,
+    "TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16": ModelType.FT,
+    "TheBloke/Wizard-Vicuna-30B-Uncensored-fp16": ModelType.FT,
+    "TheBloke/wizard-vicuna-13B-HF": ModelType.IFT,
+    "TheBloke/UltraLM-13B-fp16": ModelType.IFT,
+    "TheBloke/OpenAssistant-FT-7-Llama-30B-HF": ModelType.FT,
+    "TheBloke/vicuna-13B-1.1-HF": ModelType.IFT,
+    "TheBloke/guanaco-13B-HF": ModelType.FT,
+    "TheBloke/guanaco-65B-HF": ModelType.FT,
+    "TheBloke/airoboros-7b-gpt4-fp16": ModelType.IFT,
+    "TheBloke/llama-30b-supercot-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/Llama-2-13B-fp16": ModelType.PT,
+    "TheBloke/llama-2-70b-Guanaco-QLoRA-fp16": ModelType.FT,
+    "TheBloke/landmark-attention-llama7b-fp16": ModelType.IFT,
+    "TheBloke/Planner-7B-fp16": ModelType.IFT,
+    "TheBloke/Wizard-Vicuna-13B-Uncensored-HF": ModelType.FT,
+    "TheBloke/gpt4-alpaca-lora-13B-HF": ModelType.IFT,
+    "TheBloke/gpt4-x-vicuna-13B-HF": ModelType.IFT,
+    "TheBloke/gpt4-alpaca-lora_mlp-65B-HF": ModelType.IFT,
+    "TheBloke/tulu-13B-fp16": ModelType.IFT,
+    "TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16": ModelType.IFT,
+    "TheBloke/Llama-2-70B-fp16": ModelType.IFT,
+    "TheBloke/WizardLM-30B-fp16": ModelType.IFT,
+    "TheBloke/robin-13B-v2-fp16": ModelType.FT,
+    "TheBloke/robin-33B-v2-fp16": ModelType.FT,
+    "TheBloke/Vicuna-13B-CoT-fp16": ModelType.IFT,
+    "TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16": ModelType.FT,
+    "TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/GPlatty-30B-SuperHOT-8K-fp16": ModelType.FT,
+    "TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16": ModelType.IFT,
+    "TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16": ModelType.IFT,
+    "jphme/orca_mini_v2_ger_7b": ModelType.IFT,
+    "Ejafa/vicuna_7B_vanilla_1.1": ModelType.FT,
+    "kevinpro/Vicuna-13B-CoT": ModelType.IFT,
+    "AlekseyKorshuk/pygmalion-6b-vicuna-chatml": ModelType.FT,
+    "AlekseyKorshuk/chatml-pyg-v1": ModelType.FT,
+    "concedo/Vicuzard-30B-Uncensored": ModelType.FT,
+    "concedo/OPT-19M-ChatSalad": ModelType.FT,
+    "concedo/Pythia-70M-ChatSalad": ModelType.FT,
+    "digitous/13B-HyperMantis": ModelType.IFT,
+    "digitous/Adventien-GPTJ": ModelType.FT,
+    "digitous/Alpacino13b": ModelType.IFT,
+    "digitous/GPT-R": ModelType.IFT,
+    "digitous/Javelin-R": ModelType.IFT,
+    "digitous/Javalion-GPTJ": ModelType.IFT,
+    "digitous/Javalion-R": ModelType.IFT,
+    "digitous/Skegma-GPTJ": ModelType.FT,
+    "digitous/Alpacino30b": ModelType.IFT,
+    "digitous/Janin-GPTJ": ModelType.FT,
+    "digitous/Janin-R": ModelType.FT,
+    "digitous/Javelin-GPTJ": ModelType.FT,
+    "SaylorTwift/gpt2_test": ModelType.PT,
+    "anton-l/gpt-j-tiny-random": ModelType.FT,
+    "Andron00e/YetAnother_Open-Llama-3B-LoRA-OpenOrca": ModelType.FT,
+    "Lazycuber/pyg-instruct-wizardlm": ModelType.FT,
+    "Lazycuber/Janemalion-6B": ModelType.FT,
+    "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1": ModelType.FT,
+    "IDEA-CCNL/Ziya-LLaMA-13B-v1": ModelType.IFT,
+    "dsvv-cair/alpaca-cleaned-llama-30b-bf16": ModelType.FT,
+    "gpt2-medium": ModelType.PT,
+    "camel-ai/CAMEL-13B-Combined-Data": ModelType.IFT,
+    "camel-ai/CAMEL-13B-Role-Playing-Data": ModelType.FT,
+    "camel-ai/CAMEL-33B-Combined-Data": ModelType.IFT,
+    "PygmalionAI/pygmalion-6b": ModelType.FT,
+    "PygmalionAI/metharme-1.3b": ModelType.IFT,
+    "PygmalionAI/pygmalion-1.3b": ModelType.FT,
+    "PygmalionAI/pygmalion-350m": ModelType.FT,
+    "PygmalionAI/pygmalion-2.7b": ModelType.FT,
+    "medalpaca/medalpaca-7b": ModelType.FT,
+    "lilloukas/Platypus-30B": ModelType.IFT,
+    "lilloukas/GPlatty-30B": ModelType.FT,
+    "mncai/chatdoctor": ModelType.FT,
+    "chaoyi-wu/MedLLaMA_13B": ModelType.FT,
+    "LoupGarou/WizardCoder-Guanaco-15B-V1.0": ModelType.IFT,
+    "LoupGarou/WizardCoder-Guanaco-15B-V1.1": ModelType.FT,
+    "hakurei/instruct-12b": ModelType.IFT,
+    "hakurei/lotus-12B": ModelType.FT,
+    "shibing624/chinese-llama-plus-13b-hf": ModelType.IFT,
+    "shibing624/chinese-alpaca-plus-7b-hf": ModelType.IFT,
+    "shibing624/chinese-alpaca-plus-13b-hf": ModelType.IFT,
+    "mosaicml/mpt-7b-instruct": ModelType.IFT,
+    "mosaicml/mpt-30b-chat": ModelType.IFT,
+    "mosaicml/mpt-7b-storywriter": ModelType.FT,
+    "mosaicml/mpt-30b-instruct": ModelType.IFT,
+    "mosaicml/mpt-7b-chat": ModelType.IFT,
+    "mosaicml/mpt-30b": ModelType.PT,
+    "Corianas/111m": ModelType.IFT,
+    "Corianas/Quokka_1.3b": ModelType.IFT,
+    "Corianas/256_5epoch": ModelType.FT,
+    "Corianas/Quokka_256m": ModelType.IFT,
+    "Corianas/Quokka_590m": ModelType.IFT,
+    "Corianas/gpt-j-6B-Dolly": ModelType.FT,
+    "Corianas/Quokka_2.7b": ModelType.IFT,
+    "cyberagent/open-calm-7b": ModelType.FT,
+    "Aspik101/Nous-Hermes-13b-pl-lora_unload": ModelType.IFT,
+    "THUDM/chatglm2-6b": ModelType.IFT,
+    "MetaIX/GPT4-X-Alpasta-30b": ModelType.IFT,
+    "NYTK/PULI-GPTrio": ModelType.PT,
+    "EleutherAI/pythia-1.3b": ModelType.PT,
+    "EleutherAI/pythia-2.8b-deduped": ModelType.PT,
+    "EleutherAI/gpt-neo-125m": ModelType.PT,
+    "EleutherAI/pythia-160m": ModelType.PT,
+    "EleutherAI/gpt-neo-2.7B": ModelType.PT,
+    "EleutherAI/pythia-1b-deduped": ModelType.PT,
+    "EleutherAI/pythia-6.7b": ModelType.PT,
+    "EleutherAI/pythia-70m-deduped": ModelType.PT,
+    "EleutherAI/gpt-neox-20b": ModelType.PT,
+    "EleutherAI/pythia-1.4b-deduped": ModelType.PT,
+    "EleutherAI/pythia-2.7b": ModelType.PT,
+    "EleutherAI/pythia-6.9b-deduped": ModelType.PT,
+    "EleutherAI/pythia-70m": ModelType.PT,
+    "EleutherAI/gpt-j-6b": ModelType.PT,
+    "EleutherAI/pythia-12b-deduped": ModelType.PT,
+    "EleutherAI/gpt-neo-1.3B": ModelType.PT,
+    "EleutherAI/pythia-410m-deduped": ModelType.PT,
+    "EleutherAI/pythia-160m-deduped": ModelType.PT,
+    "EleutherAI/polyglot-ko-12.8b": ModelType.PT,
+    "EleutherAI/pythia-12b": ModelType.PT,
+    "roneneldan/TinyStories-33M": ModelType.PT,
+    "roneneldan/TinyStories-28M": ModelType.PT,
+    "roneneldan/TinyStories-1M": ModelType.PT,
+    "roneneldan/TinyStories-8M": ModelType.PT,
+    "roneneldan/TinyStories-3M": ModelType.PT,
+    "jerryjalapeno/nart-100k-7b": ModelType.FT,
+    "lmsys/vicuna-13b-v1.3": ModelType.IFT,
+    "lmsys/vicuna-7b-v1.3": ModelType.IFT,
+    "lmsys/vicuna-13b-v1.1": ModelType.IFT,
+    "lmsys/vicuna-13b-delta-v1.1": ModelType.IFT,
+    "lmsys/vicuna-7b-delta-v1.1": ModelType.IFT,
+    "abhiramtirumala/DialoGPT-sarcastic-medium": ModelType.FT,
+    "haonan-li/bactrian-x-llama-13b-merged": ModelType.IFT,
+    "Gryphe/MythoLogic-13b": ModelType.IFT,
+    "Gryphe/MythoBoros-13b": ModelType.IFT,
+    "pillowtalks-ai/delta13b": ModelType.FT,
+    "wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard": ModelType.FT,
+    "bigscience/bloom-7b1": ModelType.PT,
+    "bigcode/tiny_starcoder_py": ModelType.PT,
+    "bigcode/starcoderplus": ModelType.FT,
+    "bigcode/gpt_bigcode-santacoder": ModelType.PT,
+    "bigcode/starcoder": ModelType.PT,
+    "Open-Orca/OpenOrca-Preview1-13B": ModelType.IFT,
+    "microsoft/DialoGPT-large": ModelType.FT,
+    "microsoft/DialoGPT-small": ModelType.FT,
+    "microsoft/DialoGPT-medium": ModelType.FT,
+    "microsoft/CodeGPT-small-py": ModelType.FT,
+    "Tincando/fiction_story_generator": ModelType.FT,
+    "Pirr/pythia-13b-deduped-green_devil": ModelType.FT,
+    "Aeala/GPT4-x-AlpacaDente2-30b": ModelType.FT,
+    "Aeala/GPT4-x-AlpacaDente-30b": ModelType.FT,
+    "Aeala/GPT4-x-Alpasta-13b": ModelType.FT,
+    "Aeala/VicUnlocked-alpaca-30b": ModelType.IFT,
+    "Tap-M/Luna-AI-Llama2-Uncensored": ModelType.FT,
+    "illuin/test-custom-llama": ModelType.FT,
+    "dvruette/oasst-llama-13b-2-epochs": ModelType.FT,
+    "dvruette/oasst-gpt-neox-20b-1000-steps": ModelType.FT,
+    "dvruette/llama-13b-pretrained-dropout": ModelType.PT,
+    "dvruette/llama-13b-pretrained": ModelType.PT,
+    "dvruette/llama-13b-pretrained-sft-epoch-1": ModelType.FT,
+    "dvruette/llama-13b-pretrained-sft-do2": ModelType.FT,
+    "dvruette/oasst-gpt-neox-20b-3000-steps": ModelType.FT,
+    "dvruette/oasst-pythia-12b-pretrained-sft": ModelType.FT,
+    "dvruette/oasst-pythia-6.9b-4000-steps": ModelType.FT,
+    "dvruette/gpt-neox-20b-full-precision": ModelType.FT,
+    "dvruette/oasst-llama-13b-1000-steps": ModelType.FT,
+    "openlm-research/open_llama_7b_700bt_preview": ModelType.PT,
+    "openlm-research/open_llama_7b": ModelType.PT,
+    "openlm-research/open_llama_7b_v2": ModelType.PT,
+    "openlm-research/open_llama_3b": ModelType.PT,
+    "openlm-research/open_llama_13b": ModelType.PT,
+    "openlm-research/open_llama_3b_v2": ModelType.PT,
+    "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged": ModelType.IFT,
+    "GeorgiaTechResearchInstitute/galpaca-30b": ModelType.IFT,
+    "GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct": ModelType.IFT,
+    "databricks/dolly-v2-7b": ModelType.IFT,
+    "databricks/dolly-v2-3b": ModelType.IFT,
+    "databricks/dolly-v2-12b": ModelType.IFT,
+    "Rachneet/gpt2-xl-alpaca": ModelType.FT,
+    "Locutusque/gpt2-conversational-or-qa": ModelType.FT,
+    "psyche/kogpt": ModelType.FT,
+    "NbAiLab/nb-gpt-j-6B-alpaca": ModelType.IFT,
+    "Mikael110/llama-2-7b-guanaco-fp16": ModelType.FT,
+    "Mikael110/llama-2-13b-guanaco-fp16": ModelType.FT,
+    "Fredithefish/CrimsonPajama": ModelType.IFT,
+    "Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K": ModelType.FT,
+    "Fredithefish/ScarletPajama-3B-HF": ModelType.FT,
+    "Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4": ModelType.IFT,
+    "acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1": ModelType.IFT,
+    "eachadea/vicuna-13b-1.1": ModelType.FT,
+    "eachadea/vicuna-7b-1.1": ModelType.FT,
+    "eachadea/vicuna-13b": ModelType.FT,
+    "openaccess-ai-collective/wizard-mega-13b": ModelType.IFT,
+    "openaccess-ai-collective/manticore-13b": ModelType.IFT,
+    "openaccess-ai-collective/manticore-30b-chat-pyg-alpha": ModelType.IFT,
+    "openaccess-ai-collective/minotaur-13b": ModelType.IFT,
+    "openaccess-ai-collective/minotaur-13b-fixed": ModelType.IFT,
+    "openaccess-ai-collective/hippogriff-30b-chat": ModelType.IFT,
+    "openaccess-ai-collective/manticore-13b-chat-pyg": ModelType.IFT,
+    "pythainlp/wangchanglm-7.5B-sft-enth": ModelType.IFT,
+    "pythainlp/wangchanglm-7.5B-sft-en-sharded": ModelType.IFT,
+    "euclaise/gpt-neox-122m-minipile-digits": ModelType.FT,
+    "stabilityai/StableBeluga1-Delta": ModelType.IFT,
+    "stabilityai/stablelm-tuned-alpha-7b": ModelType.IFT,
+    "stabilityai/StableBeluga2": ModelType.IFT,
+    "stabilityai/StableBeluga-13B": ModelType.IFT,
+    "stabilityai/StableBeluga-7B": ModelType.IFT,
+    "stabilityai/stablelm-base-alpha-7b": ModelType.PT,
+    "stabilityai/stablelm-base-alpha-3b": ModelType.PT,
+    "stabilityai/stablelm-tuned-alpha-3b": ModelType.IFT,
+    "alibidaran/medical_transcription_generator": ModelType.FT,
+    "CalderaAI/30B-Lazarus": ModelType.IFT,
+    "CalderaAI/13B-BlueMethod": ModelType.IFT,
+    "CalderaAI/13B-Ouroboros": ModelType.IFT,
+    "KoboldAI/OPT-13B-Erebus": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Janeway": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Shinen": ModelType.FT,
+    "KoboldAI/fairseq-dense-2.7B": ModelType.PT,
+    "KoboldAI/OPT-6B-nerys-v2": ModelType.FT,
+    "KoboldAI/GPT-NeoX-20B-Skein": ModelType.FT,
+    "KoboldAI/PPO_Pygway-6b-Mix": ModelType.FT,
+    "KoboldAI/fairseq-dense-6.7B": ModelType.PT,
+    "KoboldAI/fairseq-dense-125M": ModelType.PT,
+    "KoboldAI/OPT-13B-Nerybus-Mix": ModelType.FT,
+    "KoboldAI/OPT-2.7B-Erebus": ModelType.FT,
+    "KoboldAI/OPT-350M-Nerys-v2": ModelType.FT,
+    "KoboldAI/OPT-2.7B-Nerys-v2": ModelType.FT,
+    "KoboldAI/OPT-2.7B-Nerybus-Mix": ModelType.FT,
+    "KoboldAI/OPT-13B-Nerys-v2": ModelType.FT,
+    "KoboldAI/GPT-NeoX-20B-Erebus": ModelType.FT,
+    "KoboldAI/OPT-6.7B-Erebus": ModelType.FT,
+    "KoboldAI/fairseq-dense-355M": ModelType.PT,
+    "KoboldAI/OPT-6.7B-Nerybus-Mix": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Adventure": ModelType.FT,
+    "KoboldAI/OPT-350M-Erebus": ModelType.FT,
+    "KoboldAI/GPT-J-6B-Skein": ModelType.FT,
+    "KoboldAI/OPT-30B-Erebus": ModelType.FT,
+    "klosax/pythia-160m-deduped-step92k-193bt": ModelType.PT,
+    "klosax/open_llama_3b_350bt_preview": ModelType.PT,
+    "klosax/openllama-3b-350bt": ModelType.PT,
+    "klosax/pythia-70m-deduped-step44k-92bt": ModelType.PT,
+    "klosax/open_llama_13b_600bt_preview": ModelType.PT,
+    "klosax/open_llama_7b_400bt_preview": ModelType.PT,
+    "kfkas/Llama-2-ko-7b-Chat": ModelType.IFT,
+    "WeOpenML/Alpaca-7B-v1": ModelType.IFT,
+    "WeOpenML/PandaLM-Alpaca-7B-v1": ModelType.IFT,
+    "TFLai/gpt2-turkish-uncased": ModelType.FT,
+    "ehartford/WizardLM-13B-Uncensored": ModelType.IFT,
+    "ehartford/dolphin-llama-13b": ModelType.IFT,
+    "ehartford/Wizard-Vicuna-30B-Uncensored": ModelType.FT,
+    "ehartford/WizardLM-30B-Uncensored": ModelType.IFT,
+    "ehartford/Wizard-Vicuna-13B-Uncensored": ModelType.FT,
+    "ehartford/WizardLM-7B-Uncensored": ModelType.IFT,
+    "ehartford/based-30b": ModelType.FT,
+    "ehartford/Wizard-Vicuna-7B-Uncensored": ModelType.FT,
+    "wahaha1987/llama_7b_sharegpt94k_fastchat": ModelType.FT,
+    "wahaha1987/llama_13b_sharegpt94k_fastchat": ModelType.FT,
+    "OpenAssistant/oasst-sft-1-pythia-12b": ModelType.FT,
+    "OpenAssistant/stablelm-7b-sft-v7-epoch-3": ModelType.IFT,
+    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5": ModelType.FT,
+    "OpenAssistant/pythia-12b-sft-v8-2.5k-steps": ModelType.IFT,
+    "OpenAssistant/pythia-12b-sft-v8-7k-steps": ModelType.IFT,
+    "OpenAssistant/pythia-12b-pre-v8-12.5k-steps": ModelType.IFT,
+    "OpenAssistant/llama2-13b-orca-8k-3319": ModelType.IFT,
+    "junelee/wizard-vicuna-13b": ModelType.FT,
+    "BreadAi/gpt-YA-1-1_160M": ModelType.PT,
+    "BreadAi/MuseCan": ModelType.PT,
+    "BreadAi/MusePy-1-2": ModelType.PT,
+    "BreadAi/DiscordPy": ModelType.PT,
+    "BreadAi/PM_modelV2": ModelType.PT,
+    "BreadAi/gpt-Youtube": ModelType.PT,
+    "BreadAi/StoryPy": ModelType.FT,
+    "julianweng/Llama-2-7b-chat-orcah": ModelType.FT,
+    "AGI-inc/lora_moe_7b_baseline": ModelType.FT,
+    "AGI-inc/lora_moe_7b": ModelType.FT,
+    "togethercomputer/GPT-NeoXT-Chat-Base-20B": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Chat-7B-v0.1": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-7B-Base": ModelType.PT,
+    "togethercomputer/RedPajama-INCITE-7B-Instruct": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Base-3B-v1": ModelType.PT,
+    "togethercomputer/Pythia-Chat-Base-7B": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Base-7B-v0.1": ModelType.PT,
+    "togethercomputer/GPT-JT-6B-v1": ModelType.IFT,
+    "togethercomputer/GPT-JT-6B-v0": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Chat-3B-v1": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-7B-Chat": ModelType.IFT,
+    "togethercomputer/RedPajama-INCITE-Instruct-3B-v1": ModelType.IFT,
+    "Writer/camel-5b-hf": ModelType.IFT,
+    "Writer/palmyra-base": ModelType.PT,
+    "MBZUAI/LaMini-GPT-1.5B": ModelType.IFT,
+    "MBZUAI/lamini-cerebras-111m": ModelType.IFT,
+    "MBZUAI/lamini-neo-1.3b": ModelType.IFT,
+    "MBZUAI/lamini-cerebras-1.3b": ModelType.IFT,
+    "MBZUAI/lamini-cerebras-256m": ModelType.IFT,
+    "MBZUAI/LaMini-GPT-124M": ModelType.IFT,
+    "MBZUAI/lamini-neo-125m": ModelType.IFT,
+    "TehVenom/DiffMerge-DollyGPT-Pygmalion": ModelType.FT,
+    "TehVenom/PPO_Shygmalion-6b": ModelType.FT,
+    "TehVenom/Dolly_Shygmalion-6b-Dev_V8P2": ModelType.FT,
+    "TehVenom/Pygmalion_AlpacaLora-7b": ModelType.FT,
+    "TehVenom/PPO_Pygway-V8p4_Dev-6b": ModelType.FT,
+    "TehVenom/Dolly_Malion-6b": ModelType.FT,
+    "TehVenom/PPO_Shygmalion-V8p4_Dev-6b": ModelType.FT,
+    "TehVenom/ChanMalion": ModelType.FT,
+    "TehVenom/GPT-J-Pyg_PPO-6B": ModelType.IFT,
+    "TehVenom/Pygmalion-13b-Merged": ModelType.FT,
+    "TehVenom/Metharme-13b-Merged": ModelType.IFT,
+    "TehVenom/Dolly_Shygmalion-6b": ModelType.FT,
+    "TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4": ModelType.IFT,
+    "georgesung/llama2_7b_chat_uncensored": ModelType.FT,
+    "vicgalle/gpt2-alpaca": ModelType.IFT,
+    "vicgalle/alpaca-7b": ModelType.FT,
+    "vicgalle/gpt2-alpaca-gpt4": ModelType.IFT,
+    "facebook/opt-350m": ModelType.PT,
+    "facebook/opt-125m": ModelType.PT,
+    "facebook/xglm-4.5B": ModelType.PT,
+    "facebook/opt-2.7b": ModelType.PT,
+    "facebook/opt-6.7b": ModelType.PT,
+    "facebook/galactica-30b": ModelType.PT,
+    "facebook/opt-13b": ModelType.PT,
+    "facebook/opt-66b": ModelType.PT,
+    "facebook/xglm-7.5B": ModelType.PT,
+    "facebook/xglm-564M": ModelType.PT,
+    "facebook/opt-30b": ModelType.PT,
+    "golaxy/gogpt-7b": ModelType.FT,
+    "golaxy/gogpt2-7b": ModelType.FT,
+    "golaxy/gogpt-7b-bloom": ModelType.FT,
+    "golaxy/gogpt-3b-bloom": ModelType.FT,
+    "psmathur/orca_mini_v2_7b": ModelType.IFT,
+    "psmathur/orca_mini_7b": ModelType.IFT,
+    "psmathur/orca_mini_3b": ModelType.IFT,
+    "psmathur/orca_mini_v2_13b": ModelType.IFT,
+    "gpt2-xl": ModelType.PT,
+    "lxe/Cerebras-GPT-2.7B-Alpaca-SP": ModelType.FT,
+    "Monero/Manticore-13b-Chat-Pyg-Guanaco": ModelType.FT,
+    "Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b": ModelType.IFT,
+    "Monero/WizardLM-13b-OpenAssistant-Uncensored": ModelType.IFT,
+    "Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b": ModelType.IFT,
+    "jzjiao/opt-1.3b-rlhf": ModelType.FT,
+    "HuggingFaceH4/starchat-beta": ModelType.IFT,
+    "KnutJaegersberg/gpt-2-xl-EvolInstruct": ModelType.IFT,
+    "KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct": ModelType.IFT,
+    "KnutJaegersberg/galactica-orca-wizardlm-1.3b": ModelType.IFT,
+    "openchat/openchat_8192": ModelType.IFT,
+    "openchat/openchat_v2": ModelType.IFT,
+    "openchat/openchat_v2_w": ModelType.IFT,
+    "ausboss/llama-13b-supercot": ModelType.IFT,
+    "ausboss/llama-30b-supercot": ModelType.IFT,
+    "Neko-Institute-of-Science/metharme-7b": ModelType.IFT,
+    "Neko-Institute-of-Science/pygmalion-7b": ModelType.FT,
+    "SebastianSchramm/Cerebras-GPT-111M-instruction": ModelType.IFT,
+    "victor123/WizardLM-13B-1.0": ModelType.IFT,
+    "OpenBuddy/openbuddy-openllama-13b-v7-fp16": ModelType.FT,
+    "OpenBuddy/openbuddy-llama2-13b-v8.1-fp16": ModelType.FT,
+    "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16": ModelType.FT,
+    "baichuan-inc/Baichuan-7B": ModelType.PT,
+    "tiiuae/falcon-40b-instruct": ModelType.IFT,
+    "tiiuae/falcon-40b": ModelType.PT,
+    "tiiuae/falcon-7b": ModelType.PT,
+    "YeungNLP/firefly-llama-13b": ModelType.FT,
+    "YeungNLP/firefly-llama-13b-v1.2": ModelType.FT,
+    "YeungNLP/firefly-llama2-13b": ModelType.FT,
+    "YeungNLP/firefly-ziya-13b": ModelType.FT,
+    "shaohang/Sparse0.5_OPT-1.3": ModelType.FT,
+    "xzuyn/Alpacino-SuperCOT-13B": ModelType.IFT,
+    "xzuyn/MedicWizard-7B": ModelType.FT,
+    "xDAN-AI/xDAN_13b_l2_lora": ModelType.FT,
+    "beomi/KoAlpaca-Polyglot-5.8B": ModelType.FT,
+    "beomi/llama-2-ko-7b": ModelType.IFT,
+    "Salesforce/codegen-6B-multi": ModelType.PT,
+    "Salesforce/codegen-16B-nl": ModelType.PT,
+    "Salesforce/codegen-6B-nl": ModelType.PT,
+    "ai-forever/rugpt3large_based_on_gpt2": ModelType.FT,
+    "gpt2-large": ModelType.PT,
+    "frank098/orca_mini_3b_juniper": ModelType.FT,
+    "frank098/WizardLM_13B_juniper": ModelType.FT,
+    "FPHam/Free_Sydney_13b_HF": ModelType.FT,
+    "huggingface/llama-13b": ModelType.PT,
+    "huggingface/llama-7b": ModelType.PT,
+    "huggingface/llama-65b": ModelType.PT,
+    "huggingface/llama-30b": ModelType.PT,
+    "Henk717/chronoboros-33B": ModelType.IFT,
+    "jondurbin/airoboros-13b-gpt4-1.4": ModelType.IFT,
+    "jondurbin/airoboros-7b": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.1": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.2": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.3": ModelType.IFT,
+    "jondurbin/airoboros-7b-gpt4-1.4": ModelType.IFT,
+    "jondurbin/airoboros-l2-7b-gpt4-1.4.1": ModelType.IFT,
+    "jondurbin/airoboros-l2-13b-gpt4-1.4.1": ModelType.IFT,
+    "jondurbin/airoboros-l2-70b-gpt4-1.4.1": ModelType.IFT,
+    "jondurbin/airoboros-13b": ModelType.IFT,
+    "jondurbin/airoboros-33b-gpt4-1.4": ModelType.IFT,
+    "jondurbin/airoboros-33b-gpt4-1.2": ModelType.IFT,
+    "jondurbin/airoboros-65b-gpt4-1.2": ModelType.IFT,
+    "ariellee/SuperPlatty-30B": ModelType.IFT,
+    "danielhanchen/open_llama_3b_600bt_preview": ModelType.FT,
+    "cerebras/Cerebras-GPT-256M": ModelType.PT,
+    "cerebras/Cerebras-GPT-1.3B": ModelType.PT,
+    "cerebras/Cerebras-GPT-13B": ModelType.PT,
+    "cerebras/Cerebras-GPT-2.7B": ModelType.PT,
+    "cerebras/Cerebras-GPT-111M": ModelType.PT,
+    "cerebras/Cerebras-GPT-6.7B": ModelType.PT,
+    "Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf": ModelType.RL,
+    "Yhyu13/llama-30B-hf-openassitant": ModelType.FT,
+    "NousResearch/Nous-Hermes-Llama2-13b": ModelType.IFT,
+    "NousResearch/Nous-Hermes-llama-2-7b": ModelType.IFT,
+    "NousResearch/Redmond-Puffin-13B": ModelType.IFT,
+    "NousResearch/Nous-Hermes-13b": ModelType.IFT,
+    "project-baize/baize-v2-7b": ModelType.IFT,
+    "project-baize/baize-v2-13b": ModelType.IFT,
+    "LLMs/WizardLM-13B-V1.0": ModelType.FT,
+    "LLMs/AlpacaGPT4-7B-elina": ModelType.FT,
+    "wenge-research/yayi-7b": ModelType.FT,
+    "wenge-research/yayi-7b-llama2": ModelType.FT,
+    "wenge-research/yayi-13b-llama2": ModelType.FT,
+    "yhyhy3/open_llama_7b_v2_med_instruct": ModelType.IFT,
+    "llama-anon/instruct-13b": ModelType.IFT,
+    "huggingtweets/jerma985": ModelType.FT,
+    "huggingtweets/gladosystem": ModelType.FT,
+    "huggingtweets/bladeecity-jerma985": ModelType.FT,
+    "huggyllama/llama-13b": ModelType.PT,
+    "huggyllama/llama-65b": ModelType.PT,
+    "FabbriSimo01/Facebook_opt_1.3b_Quantized": ModelType.PT,
+    "upstage/Llama-2-70b-instruct": ModelType.IFT,
+    "upstage/Llama-2-70b-instruct-1024": ModelType.IFT,
+    "upstage/llama-65b-instruct": ModelType.IFT,
+    "upstage/llama-30b-instruct-2048": ModelType.IFT,
+    "upstage/llama-30b-instruct": ModelType.IFT,
+    "WizardLM/WizardLM-13B-1.0": ModelType.IFT,
+    "WizardLM/WizardLM-13B-V1.1": ModelType.IFT,
+    "WizardLM/WizardLM-13B-V1.2": ModelType.IFT,
+    "WizardLM/WizardLM-30B-V1.0": ModelType.IFT,
+    "WizardLM/WizardCoder-15B-V1.0": ModelType.IFT,
+    "gpt2": ModelType.PT,
+    "keyfan/vicuna-chinese-replication-v1.1": ModelType.IFT,
+    "nthngdy/pythia-owt2-70m-100k": ModelType.FT,
+    "nthngdy/pythia-owt2-70m-50k": ModelType.FT,
+    "quantumaikr/KoreanLM-hf": ModelType.FT,
+    "quantumaikr/open_llama_7b_hf": ModelType.FT,
+    "quantumaikr/QuantumLM-70B-hf": ModelType.IFT,
+    "MayaPH/FinOPT-Lincoln": ModelType.FT,
+    "MayaPH/FinOPT-Franklin": ModelType.FT,
+    "MayaPH/GodziLLa-30B": ModelType.IFT,
+    "MayaPH/GodziLLa-30B-plus": ModelType.IFT,
+    "MayaPH/FinOPT-Washington": ModelType.FT,
+    "ogimgio/gpt-neo-125m-neurallinguisticpioneers": ModelType.FT,
+    "layoric/llama-2-13b-code-alpaca": ModelType.FT,
+    "CobraMamba/mamba-gpt-3b": ModelType.FT,
+    "CobraMamba/mamba-gpt-3b-v2": ModelType.FT,
+    "CobraMamba/mamba-gpt-3b-v3": ModelType.FT,
+    "timdettmers/guanaco-33b-merged": ModelType.FT,
+    "elinas/chronos-33b": ModelType.IFT,
+    "heegyu/RedTulu-Uncensored-3B-0719": ModelType.IFT,
+    "heegyu/WizardVicuna-Uncensored-3B-0719": ModelType.IFT,
+    "heegyu/WizardVicuna-3B-0719": ModelType.IFT,
+    "meta-llama/Llama-2-7b-chat-hf": ModelType.RL,
+    "meta-llama/Llama-2-7b-hf": ModelType.PT,
+    "meta-llama/Llama-2-13b-chat-hf": ModelType.RL,
+    "meta-llama/Llama-2-13b-hf": ModelType.PT,
+    "meta-llama/Llama-2-70b-chat-hf": ModelType.RL,
+    "meta-llama/Llama-2-70b-hf": ModelType.PT,
+    "xhyi/PT_GPTNEO350_ATG": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-20b": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt": ModelType.FT,
+    "h2oai/h2ogpt-oig-oasst1-512-6_9b": ModelType.IFT,
+    "h2oai/h2ogpt-oasst1-512-12b": ModelType.IFT,
+    "h2oai/h2ogpt-oig-oasst1-256-6_9b": ModelType.IFT,
+    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt": ModelType.FT,
+    "h2oai/h2ogpt-oasst1-512-20b": ModelType.IFT,
+    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-en-1024-12b": ModelType.FT,
+    "h2oai/h2ogpt-gm-oasst1-multilang-1024-20b": ModelType.FT,
+    "bofenghuang/vigogne-13b-instruct": ModelType.IFT,
+    "bofenghuang/vigogne-13b-chat": ModelType.FT,
+    "bofenghuang/vigogne-2-7b-instruct": ModelType.IFT,
+    "bofenghuang/vigogne-7b-instruct": ModelType.IFT,
+    "bofenghuang/vigogne-7b-chat": ModelType.FT,
+    "Vmware/open-llama-7b-v2-open-instruct": ModelType.IFT,
+    "VMware/open-llama-0.7T-7B-open-instruct-v1.1": ModelType.IFT,
+    "ewof/koishi-instruct-3b": ModelType.IFT,
+    "gywy/llama2-13b-chinese-v1": ModelType.FT,
+    "GOAT-AI/GOAT-7B-Community": ModelType.FT,
+    "psyche/kollama2-7b": ModelType.FT,
+    "TheTravellingEngineer/llama2-7b-hf-guanaco": ModelType.FT,
+    "beaugogh/pythia-1.4b-deduped-sharegpt": ModelType.FT,
+    "augtoma/qCammel-70-x": ModelType.IFT,
+    "Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload": ModelType.IFT,
+    "anhnv125/pygmalion-6b-roleplay": ModelType.FT,
+    "64bits/LexPodLM-13B": ModelType.FT,
+}
+
+
+def model_type_from_str(type):
+    if "fine-tuned" in type or "🔶" in type:
+        return ModelType.FT
+    if "pretrained" in type or "🟢" in type:
+        return ModelType.PT
+    if "RL-tuned" in type or "🟦" in type:
+        return ModelType.RL
+    if "instruction-tuned" in type or "⭕" in type:
+        return ModelType.IFT
+    return ModelType.Unknown

src/{auto_leaderboard/load_results.py → display_models/read_results.py}

@@ -1,14 +1,13 @@
-from dataclasses import dataclass
-
-import glob
 import json
 import os
+from dataclasses import dataclass
 from typing import Dict, List, Tuple
-import dateutil
 
-from src.utils_display import AutoEvalColumn, make_clickable_model
+import dateutil
 import numpy as np
 
+from src.display_models.utils import AutoEvalColumn, make_clickable_model
+
 METRICS = ["acc_norm", "acc_norm", "acc", "mc2"]
 BENCHMARKS = ["arc:challenge", "hellaswag", "hendrycksTest", "truthfulqa:mc"]
 BENCH_TO_NAME = {
@@ -31,13 +30,15 @@ class EvalResult:
     weight_type: str = ""
 
     def to_dict(self):
+        from src.load_from_hub import is_model_on_hub
+
         if self.org is not None:
             base_model = f"{self.org}/{self.model}"
         else:
             base_model = f"{self.model}"
         data_dict = {}
 
-        data_dict["eval_name"] = self.eval_name # not a column, just a save name
+        data_dict["eval_name"] = self.eval_name  # not a column, just a save name
         data_dict["weight_type"] = self.weight_type  # not a column, just a save name
         data_dict[AutoEvalColumn.precision.name] = self.precision
         data_dict[AutoEvalColumn.model_type.name] = self.model_type
@@ -45,6 +46,9 @@ class EvalResult:
         data_dict[AutoEvalColumn.dummy.name] = base_model
         data_dict[AutoEvalColumn.revision.name] = self.revision
         data_dict[AutoEvalColumn.average.name] = sum([v for k, v in self.results.items()]) / 4.0
+        data_dict[AutoEvalColumn.still_on_hub.name] = (
+            is_model_on_hub(base_model, self.revision)[0] or base_model == "baseline"
+        )
 
         for benchmark in BENCHMARKS:
             if benchmark not in self.results.keys():
@@ -60,10 +64,9 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
     with open(json_filepath) as fp:
         data = json.load(fp)
 
-    
     for mmlu_k in ["harness|hendrycksTest-abstract_algebra|5", "hendrycksTest-abstract_algebra"]:
         if mmlu_k in data["versions"] and data["versions"][mmlu_k] == 0:
-            return None, [] # we skip models with the wrong version 
+            return None, []  # we skip models with the wrong version
 
     try:
         config = data["config"]
@@ -87,22 +90,29 @@ def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
     else:
         org = model_split[0]
         model = model_split[1]
-        result_key =  f"{org}_{model}_{model_sha}_{precision}"
+        result_key = f"{org}_{model}_{model_sha}_{precision}"
 
     eval_results = []
     for benchmark, metric in zip(BENCHMARKS, METRICS):
-        accs = np.array([v[metric] for k, v in data["results"].items() if benchmark in k])
-        if accs.size == 0:
+        accs = np.array([v.get(metric, None) for k, v in data["results"].items() if benchmark in k])
+        if accs.size == 0 or any([acc is None for acc in accs]):
             continue
         mean_acc = np.mean(accs) * 100.0
-        eval_results.append(EvalResult(
-            eval_name=result_key, org=org, model=model, revision=model_sha, results={benchmark: mean_acc}, precision=precision, #todo model_type=, weight_type=
-        ))
+        eval_results.append(
+            EvalResult(
+                eval_name=result_key,
+                org=org,
+                model=model,
+                revision=model_sha,
+                results={benchmark: mean_acc},
+                precision=precision,  # todo model_type=, weight_type=
+            )
+        )
 
     return result_key, eval_results
 
 
-def get_eval_results(is_public) -> List[EvalResult]:
+def get_eval_results() -> List[EvalResult]:
     json_filepaths = []
 
     for root, dir, files in os.walk("eval-results"):
@@ -113,11 +123,11 @@ def get_eval_results(is_public) -> List[EvalResult]:
         # Sort the files by date
         # store results by precision maybe?
         try:
-            files.sort(key=lambda x:  dateutil.parser.parse(x.split("_", 1)[-1][:-5]))
+            files.sort(key=lambda x: dateutil.parser.parse(x.split("_", 1)[-1][:-5]))
         except dateutil.parser._parser.ParserError:
             files = [files[-1]]
 
-        #up_to_date = files[-1]
+        # up_to_date = files[-1]
         for file in files:
             json_filepaths.append(os.path.join(root, file))
 
@@ -135,7 +145,7 @@ def get_eval_results(is_public) -> List[EvalResult]:
     return eval_results
 
 
-def get_eval_results_dicts(is_public=True) -> List[Dict]:
-    eval_results = get_eval_results(is_public)
+def get_eval_results_dicts() -> List[Dict]:
+    eval_results = get_eval_results()
 
     return [e.to_dict() for e in eval_results]

src/{utils_display.py → display_models/utils.py}

@@ -1,19 +1,27 @@
+import os
 from dataclasses import dataclass
 
-# These classes are for user facing column names, to avoid having to change them 
-# all around the code when a modif is needed 
+from huggingface_hub import HfApi
+
+API = HfApi()
+
+
+# These classes are for user facing column names, to avoid having to change them
+# all around the code when a modif is needed
 @dataclass
 class ColumnContent:
     name: str
-    type: str 
-    displayed_by_default: bool 
+    type: str
+    displayed_by_default: bool
     hidden: bool = False
 
+
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
+
 @dataclass(frozen=True)
-class AutoEvalColumn: # Auto evals column
+class AutoEvalColumn:  # Auto evals column
     model_type_symbol = ColumnContent("T", "str", True)
     model = ColumnContent("Model", "markdown", True)
     average = ColumnContent("Average ⬆️", "number", True)
@@ -22,15 +30,19 @@ class AutoEvalColumn: # Auto evals column
     mmlu = ColumnContent("MMLU", "number", True)
     truthfulqa = ColumnContent("TruthfulQA", "number", True)
     model_type = ColumnContent("Type", "str", False)
-    precision = ColumnContent("Precision", "str", False) #, True)
+    precision = ColumnContent("Precision", "str", False)  # , True)
     license = ColumnContent("Hub License", "str", False)
     params = ColumnContent("#Params (B)", "number", False)
     likes = ColumnContent("Hub ❤️", "number", False)
+    still_on_hub = ColumnContent("Available on the hub", "bool", False)
     revision = ColumnContent("Model sha", "str", False, False)
-    dummy = ColumnContent("model_name_for_query", "str", True) # dummy col to implement search bar (hidden by custom CSS)
+    dummy = ColumnContent(
+        "model_name_for_query", "str", True
+    )  # dummy col to implement search bar (hidden by custom CSS)
+
 
 @dataclass(frozen=True)
-class EloEvalColumn: # Elo evals column
+class EloEvalColumn:  # Elo evals column
     model = ColumnContent("Model", "markdown", True)
     gpt4 = ColumnContent("GPT-4 (all)", "number", True)
     human_all = ColumnContent("Human (all)", "number", True)
@@ -39,7 +51,7 @@ class EloEvalColumn: # Elo evals column
 
 
 @dataclass(frozen=True)
-class EvalQueueColumn: # Queue column
+class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
     revision = ColumnContent("revision", "str", True)
     private = ColumnContent("private", "bool", True)
@@ -47,7 +59,13 @@ class EvalQueueColumn: # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
-LLAMAS = ["huggingface/llama-7b", "huggingface/llama-13b", "huggingface/llama-30b", "huggingface/llama-65b"]
+
+LLAMAS = [
+    "huggingface/llama-7b",
+    "huggingface/llama-13b",
+    "huggingface/llama-30b",
+    "huggingface/llama-65b",
+]
 
 
 KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
@@ -84,16 +102,45 @@ def make_clickable_model(model_name):
         link = KOALA_LINK
     elif model_name == "oasst-12b":
         link = OASST_LINK
-    #else:
-    #    link = MODEL_PAGE
-  
-    return model_hyperlink(link, model_name)
+
+    details_model_name = model_name.replace("/", "__")
+    details_link = f"https://huggingface.co/datasets/open-llm-leaderboard/details_{details_model_name}"
+
+    if not bool(os.getenv("DEBUG", "False")):
+        # We only add these checks when not debugging, as they are extremely slow
+        print(f"details_link: {details_link}")
+        try:
+            check_path = list(
+                API.list_files_info(
+                    repo_id=f"open-llm-leaderboard/details_{details_model_name}",
+                    paths="README.md",
+                    repo_type="dataset",
+                )
+            )
+            print(f"check_path: {check_path}")
+        except Exception as err:
+            # No details repo for this model
+            print(f"No details repo for this model: {err}")
+            return model_hyperlink(link, model_name)
+
+    return model_hyperlink(link, model_name) + "  " + model_hyperlink(details_link, "📑")
+
 
 def styled_error(error):
     return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
 
+
 def styled_warning(warn):
     return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
 
+
 def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/init.py

@@ -1,58 +0,0 @@
-import os 
-from huggingface_hub import Repository
-
-H4_TOKEN = os.environ.get("H4_TOKEN", None)
-
-
-def get_all_requested_models(requested_models_dir):
-    depth = 1
-    file_names = []
-
-    for root, dirs, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            file_names.extend([os.path.join(root, file) for file in files])
-
-    return set([file_name.lower().split("eval-queue/")[1] for file_name in file_names])
-
-def load_all_info_from_hub(QUEUE_REPO, RESULTS_REPO, QUEUE_PATH, RESULTS_PATH):
-    eval_queue_repo = None
-    eval_results_repo = None
-    requested_models = None
-
-    if H4_TOKEN:
-        print("Pulling evaluation requests and results.")
-
-        eval_queue_repo = Repository(
-            local_dir=QUEUE_PATH,
-            clone_from=QUEUE_REPO,
-            use_auth_token=H4_TOKEN,
-            repo_type="dataset",
-        )
-        eval_queue_repo.git_pull()
-
-        eval_results_repo = Repository(
-            local_dir=RESULTS_PATH,
-            clone_from=RESULTS_REPO,
-            use_auth_token=H4_TOKEN,
-            repo_type="dataset",
-        )
-        eval_results_repo.git_pull()
-
-        requested_models = get_all_requested_models("eval-queue")
-    else:
-        print("No HuggingFace token provided. Skipping evaluation requests and results.")
-
-    return eval_queue_repo, requested_models, eval_results_repo
-
-
-#def load_results(model, benchmark, metric):
-#    file_path = os.path.join("autoevals", model, f"{model}-eval_{benchmark}.json")
-#    if not os.path.exists(file_path):
-#        return 0.0, None
-
-#    with open(file_path) as fp:
-#        data = json.load(fp)
-#    accs = np.array([v[metric] for k, v in data["results"].items()])
-#    mean_acc = np.mean(accs)
-#    return mean_acc, data["config"]["model_args"]

src/load_from_hub.py

@@ -0,0 +1,151 @@
+import json
+import os
+
+import pandas as pd
+from huggingface_hub import Repository
+from transformers import AutoConfig
+from collections import defaultdict
+
+from src.assets.hardcoded_evals import baseline, gpt4_values, gpt35_values
+from src.display_models.get_model_metadata import apply_metadata
+from src.display_models.read_results import get_eval_results_dicts, make_clickable_model
+from src.display_models.utils import AutoEvalColumn, EvalQueueColumn, has_no_nan_values
+
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+
+
+def get_all_requested_models(requested_models_dir: str) -> set[str]:
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"): continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates
+
+
+def load_all_info_from_hub(QUEUE_REPO: str, RESULTS_REPO: str, QUEUE_PATH: str, RESULTS_PATH: str) -> list[Repository]:
+    eval_queue_repo = None
+    eval_results_repo = None
+    requested_models = None
+
+    print("Pulling evaluation requests and results.")
+
+    eval_queue_repo = Repository(
+        local_dir=QUEUE_PATH,
+        clone_from=QUEUE_REPO,
+        repo_type="dataset",
+    )
+    eval_queue_repo.git_pull()
+
+    eval_results_repo = Repository(
+        local_dir=RESULTS_PATH,
+        clone_from=RESULTS_REPO,
+        repo_type="dataset",
+    )
+    eval_results_repo.git_pull()
+
+    requested_models, users_to_submission_dates = get_all_requested_models("eval-queue")
+
+    return eval_queue_repo, requested_models, eval_results_repo, users_to_submission_dates
+
+
+def get_leaderboard_df(
+    eval_results: Repository, eval_results_private: Repository, cols: list, benchmark_cols: list
+) -> pd.DataFrame:
+    if eval_results:
+        print("Pulling evaluation results for the leaderboard.")
+        eval_results.git_pull()
+    if eval_results_private:
+        print("Pulling evaluation results for the leaderboard.")
+        eval_results_private.git_pull()
+
+    all_data = get_eval_results_dicts()
+
+    if not IS_PUBLIC:
+        all_data.append(gpt4_values)
+        all_data.append(gpt35_values)
+
+    all_data.append(baseline)
+    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
+
+    df = pd.DataFrame.from_records(all_data)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return df
+
+
+def get_evaluation_queue_df(
+    eval_queue: Repository, eval_queue_private: Repository, save_path: str, cols: list
+) -> list[pd.DataFrame]:
+    if eval_queue:
+        print("Pulling changes for the evaluation queue.")
+        eval_queue.git_pull()
+    if eval_queue_private:
+        print("Pulling changes for the evaluation queue.")
+        eval_queue_private.git_pull()
+
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]
+
+
+def is_model_on_hub(model_name: str, revision: str) -> bool:
+    try:
+        AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False)
+        return True, None
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+        )
+
+    except Exception as e:
+        print(f"Could not get the model config from the hub.: {e}")
+        return False, "was not found on hub!"

src/rate_limiting.py

@@ -0,0 +1,16 @@
+
+from datetime import datetime, timezone, timedelta
+
+
+def user_submission_permission(submission_name, users_to_submission_dates, rate_limit_period):
+    org_or_user, _ = submission_name.split("/")
+    if org_or_user not in users_to_submission_dates:
+        return 0
+    submission_dates = sorted(users_to_submission_dates[org_or_user])
+
+    time_limit = (datetime.now(timezone.utc) - timedelta(days=rate_limit_period)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    submissions_after_timelimit = [d for d in submission_dates if d > time_limit]
+
+    return len(submissions_after_timelimit)
+
+