src/about.py

from dataclasses import dataclass
from enum import Enum

@dataclass
class Task:
    benchmark: str
    metric: str
    metric_type: str
    col_name: str

# Select your tasks here
# ---------------------------------------------------
class Tasks(Enum):
    # task_key in the json file, metric_key in the json file, name to display in the leaderboard

    task1 = Task("text-entailment_1", "acc", "CPS", "TE")
    task2 = Task("text-entailment_2", "acc", "average_accuracy", "TE Prompt Average")
    task3 = Task("text-entailment_3", "acc", "best_prompt", "TE Best Prompt")
    task4 = Task("text-entailment_4", "acc", "prompt_id", "TE Best Prompt Id")

    task5 = Task("sentiment-analysis_1", "acc", "CPS", "SA")
    task6 = Task("sentiment-analysis_2", "acc", "average_accuracy", "SA Prompt Average")
    task7 = Task("sentiment-analysis_3", "acc", "best_prompt", "SA Best Prompt")
    task8 = Task("sentiment-analysis_4", "acc", "prompt_id", "SA Best Prompt Id")

    task9 = Task("hate-speech-detection_1", "acc", "CPS", "HS")
    task10 = Task("hate-speech-detection_2", "acc", "average_accuracy", "HS Prompt Average")
    task11 = Task("hate-speech-detection_3", "acc", "best_prompt", "HS Best Prompt")
    task12 = Task("hate-speech-detection_4", "acc", "prompt_id", "HS Best Prompt Id")

    task13 = Task("admission-test_1", "acc", "CPS", "AT")
    task14 = Task("admission-test_2", "acc", "average_accuracy", "AT Prompt Average")
    task15 = Task("admission-test_3", "acc", "best_prompt", "AT Best Prompt")
    task16 = Task("admission-test_4", "acc", "prompt_id", "AT Best Prompt Id")

    task17 = Task("word-in-context_1", "acc", "CPS", "WIC")
    task18 = Task("word-in-context_2", "acc", "average_accuracy", "WIC Prompt Average")
    task19 = Task("word-in-context_3", "acc", "best_prompt", "WIC Best Prompt")
    task20 = Task("word-in-context_4", "acc", "prompt_id", "WIC Best Prompt Id")

    task21 = Task("faq_1", "acc", "CPS", "FAQ")
    task22 = Task("faq_2", "acc", "average_accuracy", "FAQ Prompt Average")
    task23 = Task("faq_3", "acc", "best_prompt", "FAQ Best Prompt")
    task24 = Task("faq_4", "acc", "prompt_id", "FAQ Best Prompt Id")

    task25 = Task("lexical-substitution_1", "acc", "CPS", "LS")
    task26 = Task("lexical-substitution_2", "acc", "average_accuracy", "LS Prompt Average")
    task27 = Task("lexical-substitution_3", "acc", "best_prompt", "LS Best Prompt")
    task28 = Task("lexical-substitution_4", "acc", "prompt_id", "LS Best Prompt Id")

    task29 = Task("summarization-fanpage_1", "acc", "CPS", "SU")
    task30 = Task("summarization-fanpage_2", "acc", "average_accuracy", "SU Prompt Average")
    task31 = Task("summarization-fanpage_3", "acc", "best_prompt", "SU Best Prompt")
    task32 = Task("summarization-fanpage_4", "acc", "prompt_id", "SU Best Prompt Id")

    task33 = Task("evalita NER_1", "acc", "CPS", "NER")
    task34 = Task("evalita NER_2", "acc", "average_accuracy", "NER Prompt Average")
    task35 = Task("evalita NER_3", "acc", "best_prompt", "NER Best Prompt")
    task36 = Task("evalita NER_4", "acc", "prompt_id", "NER Best Prompt Id")

    task37 = Task("relation-extraction_1", "acc", "CPS", "REL")
    task38 = Task("relation-extraction_2", "acc", "average_accuracy", "REL Prompt Average")
    task39 = Task("relation-extraction_3", "acc", "best_prompt", "REL Best Prompt")
    task40 = Task("relation-extraction_4", "acc", "prompt_id", "REL Best Prompt Id")

    '''
    task0 = Task("TextualEntailment", "acc", "Textual Entailment")
    task1 = Task("TextualEntailment_best", "acc", "TextualEntailment Best")
    task2 = Task("Sentiment Analysis", "acc", "Sentiment Analysis")
    task3 = Task("Sentiment Analysis_best", "acc", "Sentiment Analysis_best")
    task4 = Task("Hate Speech", "acc", "Hate Speech")
    task5 = Task("Hate Speech_best", "acc", "Hate Speech_best")
    task6 = Task("Admission Test", "acc", "Admission Test")
    task7 = Task("Admission Test_best", "acc", "Admission Test_best")
    task8 = Task("Word in Context", "acc", "Word in Context")
    task9 = Task("Word in Context_best", "acc", "Word in Context_best")
    task10 = Task("FAQ", "acc", "FAQ")
    task11 = Task("FAQ_best", "acc", "FAQ_best")
    task12 = Task("Lexical Substitution", "acc", "Lexical Substitution")
    task13 = Task("Lexical Substitution_best", "acc", "Lexical Substitution_best")
    task14 = Task("Summarization", "acc", "Summarization")
    task15 = Task("Summarization_best", "acc", "Summarization_best")
    task16 = Task("NER", "acc", "NER")
    task17 = Task("NER_best", "acc", "NER_best")
    task18 = Task("REL", "acc", "REL")
    task19 = Task("REL_best", "acc", "REL_best")
    '''

# Your leaderboard name
TITLE = """<h1 align="center" id="space-title">🚀 EVALITA-LLM Leaderboard 🚀</h1>"""

# What does your leaderboard evaluate?
INTRODUCTION_TEXT = """
Evalita-LLM is a benchmark designed to evaluate Large Language Models (LLMs) on Italian tasks. The distinguishing features of Evalita-LLM are the following: (i) **all tasks are native Italian**, avoiding translation issues and potential cultural biases; (ii) the benchmark includes **generative** tasks, enabling more natural interaction with LLMs; (iii) **all tasks are evaluated against multiple prompts**, this way mitigating the model sensitivity to specific prompts and allowing a fairer evaluation.
 
**<small>Multiple-choice tasks:</small>** <small> 📊TE (Textual Entailment), 😃SA (Sentiment Analysis), ⚠️HS (Hate Speech Detection), 🏥AT (Admission Test), 🔤WIC (Word in Context), ❓FAQ (Frequently Asked Questions) </small><br>
**<small>Generative tasks:</small>** <small>🔄LS (Lexical Substitution), 📝SU (Summarization), 🏷️NER (Named Entity Recognition), 🔗REL (Relation Extraction) </small>
"""

# Which evaluations are you running? how can people reproduce what you have?
LLM_BENCHMARKS_TEXT = f"""
### Groups

- `evalita-mp`: All tasks (perplexity and non-perplexity based).
- `evalita-mp_gen`: Only generative tasks.
- `evalita-mp_mc`: Only multiple-choice tasks.

#### Tasks

The following Evalita-LLM tasks can also be evaluated in isolation:
 - `evalita-mp_te`: Textual Entailment (TE)
 - `evalita-mp_sa`: Sentiment Analysis (SA)
 - `evalita-mp_wic`: Word in Context (WIC)
 - `evalita-mp_hs`: Hate Speech Detection (HS)
 - `evalita-mp_at`: Admission Tests (AT)
 - `evalita-mp_faq`: Frequently Asked Questions & Question Answering (FAQ) 
 - `evalita-mp_sum_fp`:  Summarization (SU)
 - `evalita-mp_ls`: Lexical Substitution LS)
 - `evalita-mp_ner_group`: Named Entity Recognition (NER)
 - `evalita-mp_re`: Relation Extraction (REL)


### Usage

```bash

lm_eval --model hf --model_args pretrained=meta-llama/Llama-2-7b-hf --tasks evalita-mp --device cuda:0 --batch_size 1 
```

<!-- 
### Checklist

* [x] Is the task an existing benchmark in the literature?
 * [x] Have you referenced the original paper that introduced the task?
 * [x] If yes, does the original paper provide a reference implementation?
   * [x] Yes, original implementation contributed by author of the benchmark

If other tasks on this dataset are already supported:
* [x] Is the "Main" variant of this task clearly denoted?
* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates?
* [x] Have you noted which, if any, published evaluation setups are matched by this variant?
-->


"""

EVALUATION_QUEUE_TEXT = """
## Some good practices before submitting a model

### 1) Make sure you can load your model and tokenizer using AutoClasses:
```python
from transformers import AutoConfig, AutoModel, AutoTokenizer
config = AutoConfig.from_pretrained("your model name", revision=revision)
model = AutoModel.from_pretrained("your model name", revision=revision)
tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
```
If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.

Note: make sure your model is public!
Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!

### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!

### 3) Make sure your model has an open license!
This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗

### 4) Fill up your model card
When we add extra information about models to the leaderboard, it will be automatically taken from the model card

## In case of model failure
If your model is displayed in the `FAILED` category, its execution stopped.
Make sure you have followed the above steps first.
If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
"""

CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
CITATION_BUTTON_TEXT = r"""
@misc{magnini2025evalitallmbenchmarkinglargelanguage,
     title={Evalita-LLM: Benchmarking Large Language Models on Italian}, 
     author={Bernardo Magnini and Roberto Zanoli and Michele Resta and Martin Cimmino and Paolo Albano and Marco Madeddu and Viviana Patti},
     year={2025},
     eprint={2502.02289},
     archivePrefix={arXiv},
     primaryClass={cs.CL},
     url={https://arxiv.org/abs/2502.02289}, 
}
"""