Add information to Task

src/about.py

@@ -2,78 +2,107 @@ from dataclasses import dataclass
 from enum import Enum
 
 
+class TaskType(Enum):
+    ALL = "ALL"
+    NLI = "NLI"
+    QA = "QA"
+    RC = "RC"
+    MC = "MC"
+    EL = "EL"
+    FA = "FA"
+    MR = "MR"
+    MT = "MT"
+    STS = "STS"
+    HE = "HE"
+    CG = "CG"
+    SUM = "SUM"
+    NotTask = "?"
+
+
 @dataclass
 class Task:
     benchmark: str
     metric: str
     col_name: str
+    task_type: TaskType
+    average: bool = False
 
 
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    AVG = Task("scores", "AVG", "AVG")
-    CG = Task("scores", "CG", "CG - コード生成")  # Code Generation - コード生成
-    EL = Task("scores", "EL", "EL - エンティティリンキング")  # Entity Linking - エンティティリンキング
-    FA = Task("scores", "FA", "FA - 基礎分析")  # Fundamental Analysis - 基礎分析
-    HE = Task("scores", "HE", "HE - 言語理解")  # Human Examination - 言語理解
-    MC = Task("scores", "MC", "MC - 多肢選択式質問応答")  # Multiple Choice question answering - 多肢選択式質問応答
-    MR = Task("scores", "MR", "MR - 数学的推論")  # Mathematical Reasoning - 数学的推論
-    MT = Task("scores", "MT", "MT - 機械翻訳")  # Machine Translation - 機械翻訳
-    NLI = Task("scores", "NLI", "NLI - 自然言語推論")  # Natural Language Inference - 自然言語推論
-    QA = Task("scores", "QA", "QA - 質問応答")  # Question Answering - 質問応答
-    RC = Task("scores", "RC", "RC - 読解力")  # Reading Comprehension - 読解力
-    SUM = Task("scores", "SUM", "SUM - 要約")  # Summarization - 要約
-    alt_e_to_j_bert_score_ja_f1 = Task("scores", "alt-e-to-j_bert_score_ja_f1", "ALT E to J BERT Score")
-    alt_e_to_j_bleu_ja = Task("scores", "alt-e-to-j_bleu_ja", "ALT E to J BLEU")
-    alt_e_to_j_comet_wmt22 = Task("scores", "alt-e-to-j_comet_wmt22", "ALT E to J COMET WMT22")
-    alt_j_to_e_bert_score_en_f1 = Task("scores", "alt-j-to-e_bert_score_en_f1", "ALT J to E BERT Score")
-    alt_j_to_e_bleu_en = Task("scores", "alt-j-to-e_bleu_en", "ALT J to E BLEU")
-    alt_j_to_e_comet_wmt22 = Task("scores", "alt-j-to-e_comet_wmt22", "ALT J to E COMET WMT22")
-    chabsa_set_f1 = Task("scores", "chabsa_set_f1", "ChABSA")
-    commonsensemoralja_exact_match = Task("scores", "commonsensemoralja_exact_match", "CommonSenseMoralJA")
-    jamp_exact_match = Task("scores", "jamp_exact_match", "JAMP")
-    janli_exact_match = Task("scores", "janli_exact_match", "JANLI")
-    jcommonsenseqa_exact_match = Task("scores", "jcommonsenseqa_exact_match", "JCommonSenseQA")
-    jemhopqa_char_f1 = Task("scores", "jemhopqa_char_f1", "JEMHopQA")
-    jmmlu_exact_match = Task("scores", "jmmlu_exact_match", "JMMLU")
-    jnli_exact_match = Task("scores", "jnli_exact_match", "JNLI")
-    jsem_exact_match = Task("scores", "jsem_exact_match", "JSEM")
-    jsick_exact_match = Task("scores", "jsick_exact_match", "JSICK")
-    jsquad_char_f1 = Task("scores", "jsquad_char_f1", "JSquad")
+    AVG = Task("scores", "AVG", "AVG", TaskType.ALL, True)
+    CG = Task("scores", "CG", "CG - コード生成", TaskType.CG, True)  # Code Generation - コード生成
+    EL = Task(
+        "scores", "EL", "EL - エンティティリンキング", TaskType.EL, True
+    )  # Entity Linking - エンティティリンキング
+    FA = Task("scores", "FA", "FA - 基礎分析", TaskType.FA, True)  # Fundamental Analysis - 基礎分析
+    HE = Task("scores", "HE", "HE - 言語理解", TaskType.HE, True)  # Human Examination - 言語理解
+    MC = Task(
+        "scores", "MC", "MC - 多肢選択式質問応答", TaskType.MC, True
+    )  # Multiple Choice question answering - 多肢選択式質問応答
+    MR = Task("scores", "MR", "MR - 数学的推論", TaskType.MR, True)  # Mathematical Reasoning - 数学的推論
+    MT = Task("scores", "MT", "MT - 機械翻訳", TaskType.MT, True)  # Machine Translation - 機械翻訳
+    NLI = Task("scores", "NLI", "NLI - 自然言語推論", TaskType.NLI, True)  # Natural Language Inference - 自然言語推論
+    QA = Task("scores", "QA", "QA - 質問応答", TaskType.QA, True)  # Question Answering - 質問応答
+    RC = Task("scores", "RC", "RC - 読解力", TaskType.RC, True)  # Reading Comprehension - 読解力
+    SUM = Task("scores", "SUM", "SUM - 要約", TaskType.SUM, True)  # Summarization - 要約
+    alt_e_to_j_bert_score_ja_f1 = Task("scores", "alt-e-to-j_bert_score_ja_f1", "ALT E to J BERT Score", TaskType.MT)
+    alt_e_to_j_bleu_ja = Task("scores", "alt-e-to-j_bleu_ja", "ALT E to J BLEU", TaskType.MT)
+    alt_e_to_j_comet_wmt22 = Task("scores", "alt-e-to-j_comet_wmt22", "ALT E to J COMET WMT22", TaskType.MT)
+    alt_j_to_e_bert_score_en_f1 = Task("scores", "alt-j-to-e_bert_score_en_f1", "ALT J to E BERT Score", TaskType.MT)
+    alt_j_to_e_bleu_en = Task("scores", "alt-j-to-e_bleu_en", "ALT J to E BLEU", TaskType.MT)
+    alt_j_to_e_comet_wmt22 = Task("scores", "alt-j-to-e_comet_wmt22", "ALT J to E COMET WMT22", TaskType.MT)
+    chabsa_set_f1 = Task("scores", "chabsa_set_f1", "ChABSA", TaskType.EL)
+    commonsensemoralja_exact_match = Task(
+        "scores", "commonsensemoralja_exact_match", "CommonSenseMoralJA", TaskType.MC
+    )
+    jamp_exact_match = Task("scores", "jamp_exact_match", "JAMP", TaskType.NLI)
+    janli_exact_match = Task("scores", "janli_exact_match", "JANLI", TaskType.NLI)
+    jcommonsenseqa_exact_match = Task("scores", "jcommonsenseqa_exact_match", "JCommonSenseQA", TaskType.MC)
+    jemhopqa_char_f1 = Task("scores", "jemhopqa_char_f1", "JEMHopQA", TaskType.QA)
+    jmmlu_exact_match = Task("scores", "jmmlu_exact_match", "JMMLU", TaskType.HE)
+    jnli_exact_match = Task("scores", "jnli_exact_match", "JNLI", TaskType.NLI)
+    jsem_exact_match = Task("scores", "jsem_exact_match", "JSEM", TaskType.NLI)
+    jsick_exact_match = Task("scores", "jsick_exact_match", "JSICK", TaskType.NLI)
+    jsquad_char_f1 = Task("scores", "jsquad_char_f1", "JSquad", TaskType.RC)
     jsts_pearson = Task(
-        "scores", "jsts_pearson", "JSTS (Pearson) - 意味的類似度"
+        "scores", "jsts_pearson", "JSTS (Pearson) - 意味的類似度", TaskType.STS
     )  # Semantic Textual Similarity - 意味的類似度
     jsts_spearman = Task(
-        "scores", "jsts_spearman", "JSTS (Spearman) - 意味的類似度"
+        "scores", "jsts_spearman", "JSTS (Spearman) - 意味的類似度", TaskType.STS
     )  # Semantic Textual Similarity - 意味的類似度
-    kuci_exact_match = Task("scores", "kuci_exact_match", "KUCI")
-    mawps_exact_match = Task("scores", "mawps_exact_match", "MAWPS")
-    mbpp_code_exec = Task("scores", "mbpp_code_exec", "MBPP(exec)")
-    mbpp_pylint_check = Task("scores", "mbpp_pylint_check", "MBPP(pylint)")
-    mmlu_en_exact_match = Task("scores", "mmlu_en_exact_match", "MMLU")
-    niilc_char_f1 = Task("scores", "niilc_char_f1", "NIILC")
-    wiki_coreference_set_f1 = Task("scores", "wiki_coreference_set_f1", "Wiki Coreference")
-    wiki_dependency_set_f1 = Task("scores", "wiki_dependency_set_f1", "Wiki Dependency")
-    wiki_ner_set_f1 = Task("scores", "wiki_ner_set_f1", "Wiki NER")
-    wiki_pas_set_f1 = Task("scores", "wiki_pas_set_f1", "Wiki PAS")
-    wiki_reading_char_f1 = Task("scores", "wiki_reading_char_f1", "Wiki Reading")
+    kuci_exact_match = Task("scores", "kuci_exact_match", "KUCI", TaskType.MC)
+    mawps_exact_match = Task("scores", "mawps_exact_match", "MAWPS", TaskType.MR)
+    mbpp_code_exec = Task("scores", "mbpp_code_exec", "MBPP(exec)", TaskType.CG)
+    mbpp_pylint_check = Task("scores", "mbpp_pylint_check", "MBPP(pylint)", TaskType.CG)
+    mmlu_en_exact_match = Task("scores", "mmlu_en_exact_match", "MMLU", TaskType.HE)
+    niilc_char_f1 = Task("scores", "niilc_char_f1", "NIILC", TaskType.QA)
+    wiki_coreference_set_f1 = Task("scores", "wiki_coreference_set_f1", "Wiki Coreference", TaskType.FA)
+    wiki_dependency_set_f1 = Task("scores", "wiki_dependency_set_f1", "Wiki Dependency", TaskType.FA)
+    wiki_ner_set_f1 = Task("scores", "wiki_ner_set_f1", "Wiki NER", TaskType.FA)
+    wiki_pas_set_f1 = Task("scores", "wiki_pas_set_f1", "Wiki PAS", TaskType.FA)
+    wiki_reading_char_f1 = Task("scores", "wiki_reading_char_f1", "Wiki Reading", TaskType.FA)
     wikicorpus_e_to_j_bert_score_ja_f1 = Task(
-        "scores", "wikicorpus-e-to-j_bert_score_ja_f1", "WikiCorpus E to J BERT Score"
+        "scores", "wikicorpus-e-to-j_bert_score_ja_f1", "WikiCorpus E to J BERT Score", TaskType.MT
+    )
+    wikicorpus_e_to_j_bleu_ja = Task("scores", "wikicorpus-e-to-j_bleu_ja", "WikiCorpus E to J BLEU", TaskType.MT)
+    wikicorpus_e_to_j_comet_wmt22 = Task(
+        "scores", "wikicorpus-e-to-j_comet_wmt22", "WikiCorpus E to J COMET WMT22", TaskType.MT
     )
-    wikicorpus_e_to_j_bleu_ja = Task("scores", "wikicorpus-e-to-j_bleu_ja", "WikiCorpus E to J BLEU")
-    wikicorpus_e_to_j_comet_wmt22 = Task("scores", "wikicorpus-e-to-j_comet_wmt22", "WikiCorpus E to J COMET WMT22")
     wikicorpus_j_to_e_bert_score_en_f1 = Task(
-        "scores", "wikicorpus-j-to-e_bert_score_en_f1", "WikiCorpus J to E BERT Score"
+        "scores", "wikicorpus-j-to-e_bert_score_en_f1", "WikiCorpus J to E BERT Score", TaskType.MT
+    )
+    wikicorpus_j_to_e_bleu_en = Task("scores", "wikicorpus-j-to-e_bleu_en", "WikiCorpus J to E BLEU", TaskType.MT)
+    wikicorpus_j_to_e_comet_wmt22 = Task(
+        "scores", "wikicorpus-j-to-e_comet_wmt22", "WikiCorpus J to E COMET WMT22", TaskType.MT
     )
-    wikicorpus_j_to_e_bleu_en = Task("scores", "wikicorpus-j-to-e_bleu_en", "WikiCorpus J to E BLEU")
-    wikicorpus_j_to_e_comet_wmt22 = Task("scores", "wikicorpus-j-to-e_comet_wmt22", "WikiCorpus J to E COMET WMT22")
-    xlsum_ja_bert_score_ja_f1 = Task("scores", "xlsum_ja_bert_score_ja_f1", "XL-Sum JA BERT Score")
-    xlsum_ja_bleu_ja = Task("scores", "xlsum_ja_bleu_ja", "XL-Sum JA BLEU")
-    xlsum_ja_rouge1 = Task("scores", "xlsum_ja_rouge1", "XL-Sum ROUGE1")
-    xlsum_ja_rouge2 = Task("scores", "xlsum_ja_rouge2", "XL-Sum ROUGE2")
+    xlsum_ja_bert_score_ja_f1 = Task("scores", "xlsum_ja_bert_score_ja_f1", "XL-Sum JA BERT Score", TaskType.SUM)
+    xlsum_ja_bleu_ja = Task("scores", "xlsum_ja_bleu_ja", "XL-Sum JA BLEU", TaskType.SUM)
+    xlsum_ja_rouge1 = Task("scores", "xlsum_ja_rouge1", "XL-Sum ROUGE1", TaskType.SUM)
+    xlsum_ja_rouge2 = Task("scores", "xlsum_ja_rouge2", "XL-Sum ROUGE2", TaskType.SUM)
     # xlsum_ja_rouge2_scaling = Task("scores", "xlsum_ja_rouge2_scaling", "XL-Sum JA ROUGE2 Scaling")
-    xlsum_ja_rougeLsum = Task("scores", "xlsum_ja_rougeLsum", "XL-Sum ROUGE-Lsum")
+    xlsum_ja_rougeLsum = Task("scores", "xlsum_ja_rougeLsum", "XL-Sum ROUGE-Lsum", TaskType.SUM)
 
 
 NUM_FEWSHOT = 0  # Change with your few shot

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.about import Tasks, TaskType
 
 
 def fields(raw_class):
@@ -21,6 +21,8 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
     dummy: bool = False
+    task_type: TaskType = TaskType.NotTask
+    average: bool = False
 
 
 ## Leaderboard columns
@@ -31,7 +33,15 @@ auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "ma
 # Scores
 # auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append(
+        [
+            task.name,
+            ColumnContent,
+            ColumnContent(
+                task.value.col_name, "number", True, task_type=task.value.task_type, average=task.value.average
+            ),
+        ]
+    )
 # Model information
 auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])