Changed the timing of model architecture retrieval to when submitting.

src/leaderboard/read_evals.py

@@ -8,7 +8,6 @@ import dateutil
 
 from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, Backend, ModelType, Tasks, Version, WeightType
-from src.submission.check_validity import is_model_on_hub
 
 
 @dataclass
@@ -90,15 +89,6 @@ class EvalResult:
             result_key = f"{model}_{precision}_({num_few_shots}shots)_{add_special_tokens}"
         full_model = "/".join(org_and_model)
 
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
         if "scores" not in data:
             raise KeyError(f"'scores' key not found in JSON file: {json_filepath}")
 
@@ -117,7 +107,6 @@ class EvalResult:
             results=results,
             precision=precision,
             revision=revision,
-            architecture=architecture,
             num_few_shots=num_few_shots,
             add_special_tokens=add_special_tokens,
             llm_jp_eval_version=version,
@@ -136,6 +125,7 @@ class EvalResult:
             self.likes = request.get("likes", 0)
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
+            self.architecture = request.get("architecture", "?")
         except Exception:
             print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision}")
 

src/submission/submit.py

@@ -47,7 +47,14 @@ def add_new_eval(
         revision = "main"
 
     # Is the model on the hub?
-    model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+    model_on_hub, error, model_config = is_model_on_hub(
+        model_name=model, revision=revision, token=TOKEN, test_tokenizer=True
+    )
+    architecture = "?"
+    if model_config is not None:
+        architectures = getattr(model_config, "architectures", None)
+        if architectures:
+            architecture = ";".join(architectures)
     if not model_on_hub:
         return styled_error(f'Model "{model}" {error}')
 
@@ -85,6 +92,7 @@ def add_new_eval(
         "private": False,
         "add_special_tokens": add_special_tokens,
         "llm_jp_eval_version": current_version,
+        "architecture": architecture,
     }
 
     # Check for duplicate submission