support fp32/fp16/bf16 eval.

app.py

@@ -572,7 +572,7 @@ with demo:
                     base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)",
                             visible=not IS_PUBLIC)
                     compute_type = gr.Dropdown(
-                        choices=[i.value.name for i in ComputeDtype],
+                        choices=[i.value.name for i in ComputeDtype if i.value.name != "All"],
                         label="Compute dtype",
                         multiselect=False,
                         value="float16",

src/display/utils.py

@@ -242,6 +242,9 @@ class WeightDtype(Enum):
     int4 = ModelDetails("int4")
     nf4 = ModelDetails("nf4")
     fp4 = ModelDetails("fp4")
+    fp16 = ModelDetails("float16")
+    bf16 = ModelDetails("bfloat16")
+    fp32 = ModelDetails("float32")
 
     Unknown = ModelDetails("?")
 
@@ -260,6 +263,12 @@ class WeightDtype(Enum):
             return WeightDtype.fp4
         if weight_dtype in ["All"]:
             return WeightDtype.all
+        if weight_dtype in ["float16"]:
+            return WeightDtype.fp16
+        if weight_dtype in ["bfloat16"]:
+            return WeightDtype.bf16
+        if weight_dtype in ["float32"]:
+            return WeightDtype.fp32
         return WeightDtype.Unknown
 
 class ComputeDtype(Enum):
@@ -317,8 +326,9 @@ class Precision(Enum):
     qt_2bit = ModelDetails("2bit")
     qt_3bit = ModelDetails("3bit")
     qt_4bit = ModelDetails("4bit")
-    # qt_8bit = ModelDetails("8bit")
-    # qt_GPTQ = ModelDetails("GPTQ")
+    qt_8bit = ModelDetails("8bit")
+    qt_16bit = ModelDetails("16bit")
+    qt_32bit = ModelDetails("32bit")
     Unknown = ModelDetails("?")
 
     def from_str(precision):
@@ -332,8 +342,12 @@ class Precision(Enum):
             return Precision.qt_3bit
         if precision in ["4bit"]:
             return Precision.qt_4bit
-        # if precision in ["GPTQ", "None"]:
-        #     return Precision.qt_GPTQ
+        if precision in ["8bit"]:
+            return Precision.qt_8bit
+        if precision in ["16bit"]:
+            return Precision.qt_16bit
+        if precision in ["32bit"]:
+            return Precision.qt_32bit
         return Precision.Unknown
 
 

src/leaderboard/read_evals.py

@@ -56,7 +56,7 @@ class EvalResult:
 
         # Precision
         precision = Precision.from_str(config.get("precision", "4bit"))
-        quant_type = QuantType.from_str(config.get("quant_type", "GPTQ"))
+        quant_type = QuantType.from_str(str(config.get("quant_type", "GPTQ")))
         weight_dtype = WeightDtype.from_str(data["task_info"].get("weight_dtype", "int4"))
         compute_dtype = ComputeDtype.from_str(data["task_info"].get("compute_dtype", "bfloat16"))
         # double_quant = data["quantization_config"].get("bnb_4bit_use_double_quant", False)
@@ -209,7 +209,7 @@ def get_request_file_for_model(requests_path, model_name,
             if (
                 req_content["status"] in ["Finished"]
                 and req_content["precision"] == precision.split(".")[-1]
-                and req_content["quant_type"] == quant_type
+                and str(req_content["quant_type"]) == quant_type
                 and req_content["weight_dtype"] == weight_dtype.split(".")[-1]
                 and req_content["compute_dtype"] == compute_dtype.split(".")[-1]
             ):

src/submission/check_validity.py

@@ -69,13 +69,27 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
             return True, "uses a gated model.", None
         return False, f"was not found or misconfigured on the hub! Error raised was {e.args[0]}", None
 
+
 def get_model_size(model_info: ModelInfo, precision: str):
     size_pattern = re.compile(r"(\d+\.)?\d+(b|m)")
     safetensors = None
     try:
         safetensors = get_safetensors_metadata(model_info.id)
+        num_parameters = 0
+        mem = 0
+        for key in safetensors.parameter_count:
+            if key in ["F16", "BF16"]:
+                mem += safetensors.parameter_count[key] * 2
+            else:
+                mem += safetensors.parameter_count[key] * 4
+
+            num_parameters += safetensors.parameter_count[key]
+
+        params_b = round(num_parameters / 1e9, 2)
+        size_gb = round(mem / 1e9,2)
+        return params_b, size_gb
     except Exception as e:
-        print(e)
+        print(str(e))
 
     if safetensors is not None:
         model_size = round(sum(safetensors.parameter_count.values()) / 1e9, 3)
@@ -87,9 +101,13 @@ def get_model_size(model_info: ModelInfo, precision: str):
         except AttributeError as e:
             return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
 
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.id.lower()) else 1
+    # size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.id.lower()) else 1
     # model_size = size_factor * model_size
-    return model_size
+    if precision == "16bit":
+        size_gb = model_size * 2
+    else:
+        size_gb = model_size * 4
+    return model_size, size_gb
 
 KNOWN_SIZE_FACTOR = {
     "gptq": {"4bit": 8, "8bit": 4, "2bit": 8, "3bit": 12},

src/submission/submit.py

@@ -157,11 +157,36 @@ def add_new_eval(
         weight_dtype = "int2"
 
     if quant_type is None or quant_type == "":
-        return styled_error("Please select a quantization model like GPTQ, AWQ etc.")
-
-    model_params, model_size = get_quantized_model_parameters_memory(model_info,
+        # return styled_error("Please select a quantization model like GPTQ, AWQ etc.")
+        # for eval fp32/fp16/bf16
+        quant_type = None
+
+    if quant_type is None:
+        weight_dtype = str(getattr(model_config, "torch_dtype", "float16"))
+        if weight_dtype in ["torch.float16", "float16"]:
+            weight_dtype = "float16"
+            precision = "16bit"
+        elif weight_dtype in ["torch.bfloat16", "bfloat16"]:
+            weight_dtype = "bfloat16"
+            precision = "16bit"
+        elif weight_dtype in ["torch.float32", "float32"]:
+            weight_dtype = "float32"
+            precision = "32bit"
+        else:
+            weight_dtype = "?"
+            precision = "?"
+        model_type = "original"
+        model_params, model_size = get_model_size(model_info=model_info, precision=precision)
+    else:
+        model_params, model_size = get_quantized_model_parameters_memory(model_info,
             quant_method=quant_type.lower(),
             bits=precision)
+        model_type = "quantization"
+    else:
+        model_params, model_size = get_quantized_model_parameters_memory(model_info,
+                quant_method=quant_type.lower(),
+                bits=precision)
+        model_type = "quantization"
 
     if quant_type == "llama.cpp":
         hardware = "cpu"
@@ -170,6 +195,9 @@ def add_new_eval(
     else:
         hardware = "gpu"
 
+    if compute_dtype == "?":
+        compute_dtype = "float16"
+
     eval_entry = {
         "model": model,
         "revision": revision,
@@ -187,7 +215,7 @@ def add_new_eval(
         "hardware": hardware,
         "status": "Pending",
         "submitted_time": current_time,
-        "model_type": "quantization",
+        "model_type": model_type,
         "job_id": -1,
         "job_start_time": None,
         "scripts": script