feat: adapt the data loading part

.gitignore

@@ -11,3 +11,7 @@ eval-results/
 eval-queue-bk/
 eval-results-bk/
 logs/
+
+.idea/
+.venv/
+toys/

app.py

@@ -49,11 +49,11 @@ raw_data, original_df = get_leaderboard_df(
     EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
 leaderboard_df = original_df.copy()
 
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+# (
+#     finished_eval_queue_df,
+#     running_eval_queue_df,
+#     pending_eval_queue_df,
+# ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 
 # Searching and filtering

src/about.py

@@ -1,19 +1,36 @@
 from dataclasses import dataclass
 from enum import Enum
 
+
 @dataclass
 class Task:
-    benchmark: str
-    metric: str
-    col_name: str
+    name: str  # qa, long_doc
+
+@dataclass
+class Metric:
+    name: str  # ndcg_at_1
+
+@dataclass
+class Language:
+    name: str  # en, zh
+
+
+@dataclass
+class Domain:
+    name: str  # law, wiki
+
+
+@dataclass
+class EmbeddingModel:
+    full_name: str  # jinaai/jina-embeddings-v2-en-base
+    org: str  # jinaai
+    model: str  # jina-embeddings-v2-en-base
+    size: int  # size (millions of parameters)
+    dim: int  # output dimensions
+    max_tokens: int  # max tokens
+    model_type: str  # open, proprietary, sentence transformers
 
 
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("anli_r1", "acc", "ANLI")
-    task1 = Task("logiqa", "acc_norm", "LogiQA")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/benchmarks.py

@@ -0,0 +1,131 @@
+from dataclasses import dataclass
+from enum import Enum
+
+
+def get_safe_name(name: str):
+    """Get RFC 1123 compatible safe name"""
+    name = name.replace('-', '_')
+    return ''.join(
+        character.lower()
+        for character in name
+        if (character.isalnum() or character == '_'))
+
+
+dataset_dict = {
+    "qa": {
+        "wiki": {
+            "en": ["wikipedia_20240101", ],
+            "zh": ["wikipedia_20240101", ]
+        },
+        "web": {
+            "en": ["mC4", ],
+            "zh": ["mC4", ]
+        },
+        "news": {
+            "en": ["CC-News", ],
+            "zh": ["CC-News", ]
+        },
+        "health": {
+            "en": ["PubMedQA", ],
+            "zh": ["Huatuo-26M", ]
+        },
+        "law": {
+            "en": ["pile-of-law", ],
+            "zh": ["flk_npc_gov_cn", ]
+        },
+        "finance": {
+            "en": ["Reuters-Financial", ],
+            "zh": ["FinCorpus", ]
+        },
+        "arxiv": {
+            "en": ["Arxiv", ]},
+    },
+    "long_doc": {
+        "arxiv": {
+            "en": ["gpt-3", "llama2", "llm-survey", "gemini"],
+        },
+        "book": {
+            "en": [
+                "origin-of-species_darwin",
+                "a-brief-history-of-time_stephen-hawking"
+            ]
+        },
+        "healthcare": {
+            "en": [
+                "pubmed_100K-200K_1",
+                "pubmed_100K-200K_2",
+                "pubmed_100K-200K_3",
+                "pubmed_40K-50K_5-merged",
+                "pubmed_30K-40K_10-merged"
+            ]
+        },
+        "law": {
+            "en": [
+                "lex_files_300K-400K",
+                "lex_files_400K-500K",
+                "lex_files_500K-600K",
+                "lex_files_600K-700K"
+            ]
+        }
+    }
+}
+
+metric_list = [
+    "ndcg_at_1",
+    "ndcg_at_3",
+    "ndcg_at_5",
+    "ndcg_at_10",
+    "ndcg_at_100",
+    "ndcg_at_1000",
+    "map_at_1",
+    "map_at_3",
+    "map_at_5",
+    "map_at_10",
+    "map_at_100",
+    "map_at_1000",
+    "recall_at_1",
+    "recall_at_3",
+    "recall_at_5",
+    "recall_at_10"
+    "recall_at_100",
+    "recall_at_1000",
+    "precision_at_1",
+    "precision_at_3",
+    "precision_at_5",
+    "precision_at_10",
+    "precision_at_100",
+    "precision_at_1000",
+    "mrr_at_1",
+    "mrr_at_3",
+    "mrr_at_5",
+    "mrr_at_10",
+    "mrr_at_100",
+    "mrr_at_1000"
+]
+
+
+@dataclass
+class Benchmark:
+    name: str  # [task]_[domain]_[language]_[metric], task_key in the json file,
+    metric: str  # ndcg_at_1 ,metric_key in the json file
+    col_name: str  # [domain]_[language], name to display in the leaderboard
+
+benchmark_dict = {}
+for task, domain_dict in dataset_dict.items():
+    for domain, lang_dict in domain_dict.items():
+        for lang, dataset_list in lang_dict.items():
+            if task == "qa":
+                benchmark_name = f"{task}_{domain}_{lang}"
+                benchmark_name = get_safe_name(benchmark_name)
+                col_name = f"{domain}_{lang}"
+                for metric in dataset_list:
+                    benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name)
+            elif task == "long_doc":
+                for dataset in dataset_list:
+                    col_name = f"{domain}_{lang}_{dataset}"
+                    for metric in metric_list:
+                        benchmark_name = f"{task}_{domain}_{lang}_{dataset}_{metric}"
+                        benchmark_name = get_safe_name(benchmark_name)
+                        benchmark_dict[benchmark_name] = Benchmark(benchmark_name, metric, col_name)
+
+Benchmarks = Enum('Benchmarks', benchmark_dict)

src/display/utils.py

@@ -1,9 +1,7 @@
 from dataclasses import dataclass, make_dataclass
-from enum import Enum
 
-import pandas as pd
+from src.benchmarks import Benchmarks
 
-from src.about import Tasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -11,7 +9,7 @@ def fields(raw_class):
 
 # These classes are for user facing column names,
 # to avoid having to change them all around the code
-# when a modif is needed
+# when a modification is needed
 @dataclass
 class ColumnContent:
     name: str
@@ -20,116 +18,40 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
 
+
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(
+    ["retrieval_model", ColumnContent, ColumnContent("Retrieval Model", "markdown", True, never_hidden=True)]
+)
+auto_eval_column_dict.append(
+    ["reranking_model", ColumnContent, ColumnContent("Reranking Model", "markdown", True, never_hidden=True)]
+)
+auto_eval_column_dict.append(
+    ["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)]
+)
+for benchmark in Benchmarks:
+    auto_eval_column_dict.append(
+        [benchmark.name, ColumnContent, ColumnContent(benchmark.value.col_name, "number", True)]
+    )
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
     model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = "" # emoji
-
-
-class ModelType(Enum):
-    PT = ModelDetails(name="pretrained", symbol="🟢")
-    FT = ModelDetails(name="fine-tuned", symbol="🔶")
-    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
-    RL = ModelDetails(name="RL-tuned", symbol="🟦")
-    Unknown = ModelDetails(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-    @staticmethod
-    def from_str(type):
-        if "fine-tuned" in type or "🔶" in type:
-            return ModelType.FT
-        if "pretrained" in type or "🟢" in type:
-            return ModelType.PT
-        if "RL-tuned" in type or "🟦" in type:
-            return ModelType.RL
-        if "instruction-tuned" in type or "⭕" in type:
-            return ModelType.IFT
-        return ModelType.Unknown
-
-class WeightType(Enum):
-    Adapter = ModelDetails("Adapter")
-    Original = ModelDetails("Original")
-    Delta = ModelDetails("Delta")
-
-class Precision(Enum):
-    float16 = ModelDetails("float16")
-    bfloat16 = ModelDetails("bfloat16")
-    float32 = ModelDetails("float32")
-    #qt_8bit = ModelDetails("8bit")
-    #qt_4bit = ModelDetails("4bit")
-    #qt_GPTQ = ModelDetails("GPTQ")
-    Unknown = ModelDetails("?")
-
-    def from_str(precision):
-        if precision in ["torch.float16", "float16"]:
-            return Precision.float16
-        if precision in ["torch.bfloat16", "bfloat16"]:
-            return Precision.bfloat16
-        if precision in ["float32"]:
-            return Precision.float32
-        #if precision in ["8bit"]:
-        #    return Precision.qt_8bit
-        #if precision in ["4bit"]:
-        #    return Precision.qt_4bit
-        #if precision in ["GPTQ", "None"]:
-        #    return Precision.qt_GPTQ
-        return Precision.Unknown
 
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
 COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
 
-NUMERIC_INTERVALS = {
-    "?": pd.Interval(-1, 0, closed="right"),
-    "~1.5": pd.Interval(0, 2, closed="right"),
-    "~3": pd.Interval(2, 4, closed="right"),
-    "~7": pd.Interval(4, 9, closed="right"),
-    "~13": pd.Interval(9, 20, closed="right"),
-    "~35": pd.Interval(20, 45, closed="right"),
-    "~60": pd.Interval(45, 70, closed="right"),
-    "70+": pd.Interval(70, 10000, closed="right"),
-}
+BENCHMARK_COLS = [t.value.col_name for t in Benchmarks]

src/leaderboard/read_evals.py

@@ -1,196 +1,171 @@
 import glob
 import json
-import math
-import os
+import os.path
 from dataclasses import dataclass
+from typing import List
 
-import dateutil
-import numpy as np
+import dateutil.parser._parser
 
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
+from src.display.utils import AutoEvalColumn
+from src.benchmarks import get_safe_name
 
 
 @dataclass
 class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """Full evaluation result of a single embedding model
     """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
-    model: str
-    revision: str # commit hash, "" if main
-    results: dict
-    precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
-    date: str = "" # submission date of request file
-    still_on_hub: bool = False
+    eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]_[metric]
+    retrieval_model: str
+    reranking_model: str
+    results: list  # results on all the benchmarks over different domains, languages, and datasets. Use benchmark.name as the key
+    task: str
+    metric: str
+    timestamp: str = ""  # submission timestamp
 
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        config = data.get("config")
 
-        # Precision
-        precision = Precision.from_str(config.get("model_dtype"))
-
-        # Get model and org
-        org_and_model = config.get("model_name", config.get("model_args", None))
-        org_and_model = org_and_model.split("/", 1)
-
-        if len(org_and_model) == 1:
-            org = None
-            model = org_and_model[0]
-            result_key = f"{model}_{precision.value.name}"
-        else:
-            org = org_and_model[0]
-            model = org_and_model[1]
-            result_key = f"{org}_{model}_{precision.value.name}"
-        full_model = "/".join(org_and_model)
+@dataclass
+class FullEvalResult:
+    eval_name: str  # name of the evaluation, [retrieval_model]_[reranking_model]
+    retrieval_model: str
+    reranking_model: str
+    results: List[EvalResult]  # results on all the EvalResults over different tasks and metrics.
+    date: str = ""
 
-        still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+    @classmethod
+    def init_from_json_file(cls, json_filepath):
+        """Initiate from the result json file for a single model.
+        The json file will be written only when the status is FINISHED.
+        """
+        with open(json_filepath) as fp:
+            model_data = json.load(fp)
+
+        # store all the results for different metrics and tasks
+        result_list = []
+        for item in model_data:
+            config = item.get("config", {})
+            # eval results for different metrics
+            results = item.get("results", [])
+            eval_result = EvalResult(
+                eval_name=f"{config['retrieval_model']}_{config['reranking_model']}_{config['metric']}",
+                retrieval_model=config["retrieval_model"],
+                reranking_model=config["reranking_model"],
+                results=results,
+                task=config["task"],
+                metric=config["metric"]
+            )
+            result_list.append(eval_result)
+        return cls(
+            eval_name=f"{result_list[0].retrieval_model}_{result_list[0].reranking_model}",
+            retrieval_model=result_list[0].retrieval_model,
+            reranking_model=result_list[0].reranking_model,
+            results=result_list
         )
-        architecture = "?"
-        if model_config is not None:
-            architectures = getattr(model_config, "architectures", None)
-            if architectures:
-                architecture = ";".join(architectures)
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
 
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
-            still_on_hub=still_on_hub,
-            architecture=architecture
+    def to_dict(self, task='qa', metric='ndcg_at_1'):
+        """Convert FullEvalResult to a list of dict compatible with our dataframe UI
+        """
+        results = []
+        for eval_result in self.results:
+            if eval_result.metric != metric:
+                continue
+            if eval_result.task != task:
+                continue
+            data_dict = {
+                "eval_name": eval_result.eval_name,
+                AutoEvalColumn.retrieval_model.name: self.retrieval_model,
+                AutoEvalColumn.reranking_model.name: self.reranking_model,
+            }
+            for result in eval_result.results:
+                # add result for each domain, language, and dataset
+                domain = result["domain"]
+                lang = result["lang"]
+                dataset = result["dataset"]
+                value = result["value"]
+                if task == 'qa':
+                    benchmark_name = f"{task}_{domain}_{lang}"
+                elif task == 'long_doc':
+                    benchmark_name = f"{task}_{domain}_{lang}_{dataset}_{metric}"
+                data_dict[get_safe_name(benchmark_name)] = value
+            results.append(data_dict)
+        return results
+
+    def update_with_request_file(self, request_path):
+        """
+        Update the request file
+        """
+        request_file = get_request_file_for_model(
+            request_path, self.retrieval_model, self.reranking_model
         )
 
-    def update_with_request_file(self, requests_path):
-        """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
-            self.model_type = ModelType.from_str(request.get("model_type", ""))
-            self.weight_type = WeightType[request.get("weight_type", "Original")]
-            self.license = request.get("license", "?")
-            self.likes = request.get("likes", 0)
-            self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
-            AutoEvalColumn.architecture.name: self.architecture,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
-    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+            print(f"Failed to find request file for {self.retrieval_model}, {self.reranking_model}: {request_path}")
+
+
+def get_request_file_for_model(requests_path, retrieval_model_name, reranking_model_name):
+    """
+    Load the request status from a json file
+    """
     request_files = os.path.join(
         requests_path,
-        f"{model_name}_eval_request_*.json",
+        f"{retrieval_model_name}",
+        f"{reranking_model_name}",
+        "eval_request_*.json",
     )
     request_files = glob.glob(request_files)
 
-    # Select correct request file (precision)
     request_file = ""
     request_files = sorted(request_files, reverse=True)
     for tmp_request_file in request_files:
         with open(tmp_request_file, "r") as f:
             req_content = json.load(f)
-            if (
-                req_content["status"] in ["FINISHED"]
-                and req_content["precision"] == precision.split(".")[-1]
-            ):
+            if req_content["status"] in ["FINISHED"]:
                 request_file = tmp_request_file
+                break
     return request_file
 
 
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
+def get_raw_eval_results(results_path: str, requests_path: str) -> List[FullEvalResult]:
+    """
+    Load the evaluation results from a json file
+    """
     model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
+    for root, dirs, files in os.walk(results_path):
         if len(files) == 0 or any([not f.endswith(".json") for f in files]):
             continue
-
-        # Sort the files by date
         try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_demo_")[:-7], reverse=True)
         except dateutil.parser._parser.ParserError:
             files = [files[-1]]
 
+        # select the latest and finished results
         for file in files:
             model_result_filepaths.append(os.path.join(root, file))
 
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        # create evaluation results
+        # TODO: fix the bug here, the running results should not be loaded
+        eval_result = FullEvalResult.init_from_json_file(model_result_filepath)
+        # get the latest result that is finished
         eval_result.update_with_request_file(requests_path)
-
-        # Store results of same eval together
+        latest_date_str = eval_result.date.replace(":", "-")
+        model_result_date_str = model_result_filepath.split('/')[-1
+        ].removeprefix("results_demo_").removesuffix(".json")
+        if latest_date_str != model_result_date_str:
+            continue
         eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
+        eval_results[eval_name] = eval_result
 
     results = []
-    for v in eval_results.values():
+    for k, v in eval_results.items():
         try:
-            v.to_dict() # we test if the dict version is complete
+            v.to_dict()
             results.append(v)
-        except KeyError:  # not all eval values present
+        except KeyError:
+            print(f"loading failed: {k}")
             continue
-
     return results

src/populate.py

@@ -24,7 +24,7 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 
 
 def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
+    """Creates the different dataframes for the evaluation queues requests"""
     entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
     all_evals = []
 

tests/src/display/test_utils.py

@@ -0,0 +1,15 @@
+import pytest
+from src.display.utils import fields, AutoEvalColumn, COLS, COLS_LITE, TYPES, EVAL_COLS, BENCHMARK_COLS
+
+
+def test_fields():
+    for c in fields(AutoEvalColumn):
+        print(c.name)
+
+
+def test_macro_variables():
+    print(f'COLS: {COLS}')
+    print(f'COLS_LITE: {COLS_LITE}')
+    print(f'TYPES: {TYPES}')
+    print(f'EVAL_COLS: {EVAL_COLS}')
+    print(f'BENCHMARK_COLS: {BENCHMARK_COLS}')

tests/src/leaderboard/test_read_evals.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+
+from src.leaderboard.read_evals import FullEvalResult, get_raw_eval_results, get_request_file_for_model
+
+cur_fp = Path(__file__)
+
+
+def test_init_from_json_file():
+    json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
+    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
+    assert len(full_eval_result.results) == 6
+
+
+def test_to_dict():
+    json_fp = cur_fp.parents[2] / "toydata" / "test_data.json"
+    full_eval_result = FullEvalResult.init_from_json_file(json_fp)
+    result_dict = full_eval_result.to_dict(task='qa', metric='ndcg_at_1')
+    assert len(result_dict) == 2
+
+
+def test_get_request_file_for_model():
+    requests_path = cur_fp.parents[2] / "toydata" / "test_requests"
+    request_file = get_request_file_for_model(requests_path, "bge-m3", "bge-reranker-v2-m3")
+    # only load the latest finished results
+    assert Path(request_file).name.removeprefix("eval_request_").removesuffix(".json") == "2023-11-21T18-10-08"
+
+
+def test_get_raw_eval_results():
+    requests_path = cur_fp.parents[2] / "toydata" / "test_requests"
+    results_path = cur_fp.parents[2] / "toydata" / "test_results" / "bge-m3"
+    results = get_raw_eval_results(results_path, requests_path)
+    # only load the latest results
+    assert len(results) == 2
+    assert results[0].date == "2023-12-21T18:10:08"
+    assert results[0].eval_name == "bge-m3_NoReranker"
+    assert len(results[0].results) == 3
+    assert results[1].eval_name == "bge-m3_bge-reranker-v2-m3"
+    assert results[1].date == "2023-11-21T18:10:08"
+    assert len(results[1].results) == 6

tests/src/test_populate.py

@@ -0,0 +1,12 @@
+from src.populate import get_leaderboard_df
+from pathlib import Path
+
+cur_fp = Path(__file__)
+
+def test_get_leaderboard_df():
+    requests_path = cur_fp.parents[2] / "toydata" / "test_requests"
+    results_path = cur_fp.parents[2] / "toydata" / "test_results"
+    cols = []
+    benchmark_cols = []
+    COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+    get_leaderboard_df(results_path, requests_path, cols, benchmark_cols)

tests/toydata/test_data.json

@@ -0,0 +1,98 @@
+[
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.75723
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.69909
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.69083
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.73359
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  }
+]

tests/toydata/test_requests/bge-m3/NoReranker/eval_request_2023-11-21T18-10-08.json

@@ -0,0 +1,6 @@
+{
+  "retrieval_model": "BAAI/bge-m3",
+  "reranking_model": "NoReranker",
+  "status": "FINISHED",
+  "submitted_time": "2023-11-21T18:10:08"
+}

tests/toydata/test_requests/bge-m3/NoReranker/eval_request_2023-12-21T18-10-08.json

@@ -0,0 +1,6 @@
+{
+  "retrieval_model": "BAAI/bge-m3",
+  "reranking_model": "NoReranker",
+  "status": "FINISHED",
+  "submitted_time": "2023-12-21T18:10:08"
+}

tests/toydata/test_requests/bge-m3/bge-reranker-v2-m3/eval_request_2023-11-21T18-10-08.json

@@ -0,0 +1,6 @@
+{
+  "retrieval_model": "BAAI/bge-m3",
+  "reranking_model": "BAAI/bge-reranker-v2-m3",
+  "status": "FINISHED",
+  "submitted_time": "2023-11-21T18:10:08"
+}

tests/toydata/test_requests/bge-m3/bge-reranker-v2-m3/eval_request_2023-12-21T18-10-08.json

@@ -0,0 +1,6 @@
+{
+  "retrieval_model": "BAAI/bge-m3",
+  "reranking_model": "BAAI/bge-reranker-v2-m3",
+  "status": "RUNNING",
+  "submitted_time": "2023-12-21T18:10:08"
+}

tests/toydata/test_results/bge-m3/NoReranker/results_demo_2023-11-21T18-10-08.json

@@ -0,0 +1,98 @@
+[
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "long_doc",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.75723
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "long_doc",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.69909
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.69083
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.73359
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  }
+]

tests/toydata/test_results/bge-m3/NoReranker/results_demo_2023-12-21T18-10-08.json

@@ -0,0 +1,50 @@
+[
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "long_doc",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.75723
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.69083
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "NoReranker",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  }
+]

tests/toydata/test_results/bge-m3/bge-reranker-v2-m3/results_demo_2023-11-21T18-10-08.json

@@ -0,0 +1,98 @@
+[
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.75723
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "long_doc",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "law",
+        "lang": "en",
+        "dataset": "lex_files_500K-600K",
+        "value": 0.69909
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.69083
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "en",
+        "dataset": "unknown",
+        "value": 0.73359
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_1"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  },
+  {
+    "config": {
+      "retrieval_model": "bge-m3",
+      "reranking_model": "bge-reranker-v2-m3",
+      "task": "qa",
+      "metric": "ndcg_at_3"
+    },
+    "results": [
+      {
+        "domain": "wiki",
+        "lang": "zh",
+        "dataset": "unknown",
+        "value": 0.78358
+      }
+    ]
+  }
+]