Update Space New

README.md

@@ -1,5 +1,5 @@
 ---
-title: Demo Leaderboard
+title: Alignment
 emoji: 🥇
 colorFrom: green
 colorTo: indigo
@@ -7,6 +7,7 @@ sdk: gradio
 app_file: app.py
 pinned: true
 license: apache-2.0
+short_description: alignment leader board
 ---
 
 # Start the configuration

app.py

@@ -33,13 +33,13 @@ def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 ### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
-    )
-except Exception:
-    restart_space()
+# try:
+#     print(EVAL_REQUESTS_PATH)
+#     snapshot_download(
+#         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+#     )
+# except Exception:
+#     restart_space()
 try:
     print(EVAL_RESULTS_PATH)
     snapshot_download(
@@ -51,11 +51,11 @@ except Exception:
 
 LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+# (
+#     finished_eval_queue_df,
+#     running_eval_queue_df,
+#     pending_eval_queue_df,
+# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
@@ -106,41 +106,41 @@ with demo:
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
 
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
+                # with gr.Column():
+                #     with gr.Accordion(
+                #         f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
+                #         open=False,
+                #     ):
+                #         with gr.Row():
+                #             finished_eval_table = gr.components.Dataframe(
+                #                 value=finished_eval_queue_df,
+                #                 headers=EVAL_COLS,
+                #                 datatype=EVAL_TYPES,
+                #                 row_count=5,
+                #             )
+                #     with gr.Accordion(
+                #         f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
+                #         open=False,
+                #     ):
+                #         with gr.Row():
+                #             running_eval_table = gr.components.Dataframe(
+                #                 value=running_eval_queue_df,
+                #                 headers=EVAL_COLS,
+                #                 datatype=EVAL_TYPES,
+                #                 row_count=5,
+                #             )
+
+                #     with gr.Accordion(
+                #         f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
+                #         open=False,
+                #     ):
+                #         with gr.Row():
+                #             pending_eval_table = gr.components.Dataframe(
+                #                 value=pending_eval_queue_df,
+                #                 headers=EVAL_COLS,
+                #                 datatype=EVAL_TYPES,
+                #                 row_count=5,
+                #             )
             with gr.Row():
                 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
 
@@ -199,6 +199,6 @@ with demo:
             )
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(restart_space, "interval", seconds=24*3600)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=1).launch()

requirements.txt

@@ -1,4 +1,4 @@
-APScheduler
+apscheduler
 black
 datasets
 gradio
@@ -6,11 +6,9 @@ gradio[oauth]
 gradio_leaderboard==0.0.13
 gradio_client
 huggingface-hub>=0.18.0
-matplotlib
 numpy
 pandas
 python-dateutil
 tqdm
-transformers
-tokenizers>=0.15.0
-sentencepiece
+sentencepiece
+transformers

src/about.py

@@ -12,8 +12,10 @@ class Task:
 # ---------------------------------------------------
 class Tasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
+    task0 = Task("task_name", "safty", "safty")
+    task1 = Task("task_name2", "fairness", "fairness")
+    task2 = Task("task_name3", "socail-norm", "socail-norm")
+
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -26,7 +26,7 @@ auto_eval_column_dict = []
 auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avrage", "number", True)])
 for task in Tasks:
     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information

src/envs.py

@@ -6,12 +6,12 @@ from huggingface_hub import HfApi
 # ----------------------------------
 TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "fatmerajabi11" # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
-REPO_ID = f"{OWNER}/leaderboard"
+REPO_ID = f"{OWNER}/alignment"
 QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
+RESULTS_REPO = f"{OWNER}/alignment_results"
 
 # If you setup a cache later, just change HF_HOME
 CACHE_PATH=os.getenv("HF_HOME", ".")

src/leaderboard/read_evals.py

@@ -1,15 +1,16 @@
 import glob
 import json
-import math
+# import math
 import os
 from dataclasses import dataclass
+import shutil
 
 import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
 from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
+# from src.submission.check_validity import is_model_on_hub
 
 
 @dataclass
@@ -57,14 +58,14 @@ class EvalResult:
             result_key = f"{org}_{model}_{precision.value.name}"
         full_model = "/".join(org_and_model)
 
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
-        )
+        # still_on_hub, _, model_config = is_model_on_hub(
+        #     full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        # )
         architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
+        # if model_config is not None:
+        #     architectures = getattr(model_config, "architectures", None)
+        #     if architectures:
+        #         architecture = ";".join(architectures)
 
         # Extract results available in this file (some results are split in several files)
         results = {}
@@ -78,7 +79,7 @@ class EvalResult:
 
             mean_acc = np.mean(accs) * 100.0
             results[task.benchmark] = mean_acc
-
+            
         return self(
             eval_name=result_key,
             full_model=full_model,
@@ -87,25 +88,26 @@ class EvalResult:
             results=results,
             precision=precision,  
             revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
+            # it use still_on_hub param where it comment in upper lines
+            still_on_hub=False,
             architecture=architecture
         )
 
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
-            self.date = request.get("submitted_time", "")
-        except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+    # def update_with_request_file(self, requests_path):
+    #     """Finds the relevant request file for the current model and updates info with it"""
+    #     request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+    #     try:
+    #         with open(request_file, "r") as f:
+    #             request = json.load(f)
+    #         self.model_type = ModelType.from_str(request.get("model_type", ""))
+    #         self.weight_type = WeightType[request.get("weight_type", "Original")]
+    #         self.license = request.get("license", "?")
+    #         self.likes = request.get("likes", 0)
+    #         self.num_params = request.get("params", 0)
+    #         self.date = request.get("submitted_time", "")
+    #     except Exception:
+    #         print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
@@ -132,51 +134,46 @@ class EvalResult:
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
-    request_files = os.path.join(
-        requests_path,
-        f"{model_name}_eval_request_*.json",
-    )
-    request_files = glob.glob(request_files)
-
-    # Select correct request file (precision)
-    request_file = ""
-    request_files = sorted(request_files, reverse=True)
-    for tmp_request_file in request_files:
-        with open(tmp_request_file, "r") as f:
-            req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
-                request_file = tmp_request_file
-    return request_file
+# def get_request_file_for_model(requests_path, model_name, precision):
+#     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+#     request_files = os.path.join(
+#         requests_path,
+#         f"{model_name}_eval_request_*.json",
+#     )
+#     request_files = glob.glob(request_files)
+
+#     # Select correct request file (precision)
+#     request_file = ""
+#     request_files = sorted(request_files, reverse=True)
+#     for tmp_request_file in request_files:
+#         with open(tmp_request_file, "r") as f:
+#             req_content = json.load(f)
+#             if (
+#                 req_content["status"] in ["FINISHED"]
+#                 and req_content["precision"] == precision.split(".")[-1]
+#             ):
+#                 request_file = tmp_request_file
+#     return request_file
 
 
 def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
 
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
+    files = glob.glob(os.path.join(results_path, "*.json"), recursive=True)
+    # Sort the files by date
+    try:
+        files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+    except dateutil.parser._parser.ParserError:
+        files = [files[-1]]
+    for file in files:
+        model_result_filepaths.append(file)
+            
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result
         eval_result = EvalResult.init_from_json_file(model_result_filepath)
-        eval_result.update_with_request_file(requests_path)
+        # eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together
         eval_name = eval_result.eval_name
@@ -192,5 +189,6 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
             results.append(v)
         except KeyError:  # not all eval values present
             continue
-
+        
+    shutil.rmtree(results_path) 
     return results

src/populate.py

@@ -12,11 +12,9 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
-
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df = df[cols].round(decimals=2)
-
     # filter out if any of the benchmarks have not been produced
     df = df[has_no_nan_values(df, benchmark_cols)]
     return df