Update submission

app.py

@@ -148,43 +148,13 @@ with demo:
             with gr.Row():
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
-                    model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
-                        label="Model type",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
-
-                with gr.Column():
-                    precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
-                        label="Precision",
-                        multiselect=False,
-                        value="float16",
-                        interactive=True,
-                    )
-                    weight_type = gr.Dropdown(
-                        choices=[i.value.name for i in WeightType],
-                        label="Weights type",
-                        multiselect=False,
-                        value="Original",
-                        interactive=True,
-                    )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
+                   
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()
             submit_button.click(
                 add_new_eval,
                 [
                     model_name_textbox,
-                    base_model_name_textbox,
-                    revision_name_textbox,
-                    precision,
-                    weight_type,
-                    model_type,
                 ],
                 submission_result,
             )

src/submission/check_validity.py

@@ -31,7 +31,7 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
 
     return True, ""
 
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+def is_model_on_hub(model_name: str, revision: str = "main", token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
     """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
     try:
         config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
@@ -59,14 +59,14 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
         return False, "was not found on hub!", None
 
 
-def get_model_size(model_info: ModelInfo, precision: str):
+def get_model_size(model_info: ModelInfo): #, precision: str
     """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
     try:
         model_size = round(model_info.safetensors["total"] / 1e9, 3)
     except (AttributeError, TypeError):
         return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    print("******* model size **********",model_size)
+    size_factor = 1#8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
     model_size = size_factor * model_size
     return model_size
 

src/submission/submit.py

@@ -17,11 +17,11 @@ USERS_TO_SUBMISSION_DATES = None
 
 def add_new_eval(
     model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    weight_type: str,
-    model_type: str,
+    #base_model: str,
+    #revision: str,
+    #precision: str,
+    #weight_type: str,
+    #model_type: str,
 ):
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
@@ -34,9 +34,13 @@ def add_new_eval(
         user_name = model.split("/")[0]
         model_path = model.split("/")[1]
 
-    precision = precision.split(" ")[0]
+    #precision = precision.split(" ")[0]
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
 
+    if not is_model_on_hub(model_name=model, token=TOKEN, test_tokenizer=True): #revision=revision
+        return styled_error("Model does not exist on HF Hub. Please select a valid model name.")
+    
+    """
     if model_type is None or model_type == "":
         return styled_error("Please select a model type.")
 
@@ -54,14 +58,14 @@ def add_new_eval(
         model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')
-
+    """
     # Is the model info correctly filled?
     try:
-        model_info = API.model_info(repo_id=model, revision=revision)
+        model_info = API.model_info(repo_id=model)#, revision=revision
     except Exception:
         return styled_error("Could not get your model information. Please fill it up properly.")
 
-    model_size = get_model_size(model_info=model_info, precision=precision)
+    model_size = get_model_size(model_info=model_info)#, precision=precision
 
     if model_size>30:
         return styled_error("Due to limited GPU availability, evaluations for models larger than 30B are currently not automated. Please open a ticket here so we do it manually for you. https://huggingface.co/spaces/silma-ai/Arabic-Broad-Leaderboard/discussions")
@@ -77,31 +81,32 @@ def add_new_eval(
         return styled_error(error_msg)
 
     # Seems good, creating the eval
-    print("Adding new eval")
+    print("Preparing a new eval")
 
     eval_entry = {
         "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "precision": precision,
-        "weight_type": weight_type,
+        "model_sha": model_info.sha,
+        #"base_model": base_model,
+        #"revision": revision,
+        #"precision": precision,
+        #"weight_type": weight_type,
         "status": "PENDING",
         "submitted_time": current_time,
-        "model_type": model_type,
+        #"model_type": model_type,
         "likes": model_info.likes,
         "params": model_size,
         "license": license,
-        "private": False,
+        #"private": False,
     }
 
     # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+    if f"{model}" in REQUESTED_MODELS: #_{revision}_{precision}
         return styled_warning("This model has been already submitted.")
 
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    out_path = f"{OUT_DIR}/{model_path}_eval_request.json" #_{precision}_{weight_type}
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))
@@ -109,8 +114,8 @@ def add_new_eval(
 
     ##update queue file
     queue_file_path = "./eval_queue.json"
-    print(TOKEN)
-    ## download queue_file from repo using huggun face hub API, update it and upload again
+
+    ## download queue_file from repo using HuggingFace hub API, update it and upload again
     queue_file = hf_hub_download(
         filename=queue_file_path,
         repo_id=QUEUE_REPO,
@@ -121,18 +126,24 @@ def add_new_eval(
     
     with open(queue_file, "r") as f:
         queue_data = json.load(f)
+
     if len(queue_data) == 0:
         queue_data = []
+
     queue_data.append(eval_entry)
+
     print(queue_data)
+
     #with open(queue_file, "w") as f:
     #    json.dump(queue_data, f)
-    print("Updating queue file")
+
+    print("Updating eval queue file")
     API.upload_file(
         path_or_fileobj=json.dumps(queue_data, indent=2).encode("utf-8"),
         path_in_repo=queue_file_path,
         repo_id=QUEUE_REPO,
-        repo_type="space"
+        repo_type="space",
+        commit_message=f"Add {model} to eval queue"
     )
         
 
@@ -142,17 +153,14 @@ def add_new_eval(
         path_in_repo=out_path,
         repo_id=QUEUE_REPO,
         repo_type="space",
-        commit_message=f"Add {model} to eval queue",
+        commit_message=f"Add {model} request file",
     )
 
 
-    
-    
-
-
     # Remove the local file
     os.remove(out_path)
 
+
     return styled_message(
         "Your request has been submitted to the evaluation queue!\nPlease wait for up to an 15 minutes for the model to show in the PENDING list."
     )