Spaces:
Sleeping
Sleeping
File size: 3,985 Bytes
04191fb 4af6f49 04191fb 4af6f49 04191fb 4af6f49 04191fb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 |
# 🔢 Evaluate your model
To evaluate your model according to the methodology used in our paper, you can use the following code.
```python
import os
import string
from Levenshtein import ratio
from datasets import load_dataset, Dataset, concatenate_datasets
from sklearn.metrics import classification_report, f1_score, accuracy_score
# 🔧 Change this path to where your JSONL prediction files are stored
outputs_path = "./"
_DATASETS = [
"cafe", "crema_d", "emns", "emozionalmente", "enterface",
"jl_Corpus", "mesd", "nemo", "oreau", "pavoque",
"ravdess", "resd", "subesco",
]
THRESHOLD = 0.57
def get_expected(split: str) -> tuple[set, str, dict]:
"""Load expected emotion labels and language metadata from CAMEO dataset."""
ds = load_dataset("amu-cai/CAMEO", split=split)
return set(ds["emotion"]), ds["language"][0], dict(zip(ds["file_id"], ds["emotion"]))
def process_outputs(dataset_name: str) -> tuple[Dataset, set, str]:
"""Clean and correct predictions, returning a Dataset with fixed predictions."""
outputs = Dataset.from_json(os.path.join(outputs_path, f"{dataset_name}.jsonl"))
options, language, expected = get_expected(dataset_name)
def preprocess(x):
return {
"predicted": x["predicted"].translate(str.maketrans('', '', string.punctuation)).lower().strip(),
"expected": expected.get(x["file_id"]),
}
outputs = outputs.map(preprocess)
def fix_prediction(x):
if x["predicted"] in options:
x["fixed_prediction"] = x["predicted"]
else:
predicted_words = x["predicted"].split()
label_scores = {
label: sum(r for r in (ratio(label, word) for word in predicted_words) if r > THRESHOLD)
for label in options
}
x["fixed_prediction"] = max(label_scores, key=label_scores.get)
return x
outputs = outputs.map(fix_prediction)
return outputs, options, language
def calculate_metrics(outputs: Dataset, labels: set) -> dict:
"""Compute classification metrics."""
y_true = outputs["expected"]
y_pred = outputs["fixed_prediction"]
return {
"f1_macro": f1_score(y_true, y_pred, average="macro"),
"weighted_f1": f1_score(y_true, y_pred, average="weighted"),
"accuracy": accuracy_score(y_true, y_pred),
"metrics_per_label": classification_report(
y_true, y_pred, target_names=sorted(labels), output_dict=True
),
}
# 🧮 Main Evaluation Loop
results = []
outputs_per_language = {}
full_outputs, full_labels = None, set()
for dataset in _DATASETS:
jsonl_path = os.path.join(outputs_path, f"{dataset}.jsonl")
if not os.path.isfile(jsonl_path):
print(f"Jsonl file for {dataset} not found.")
continue
outputs, labels, language = process_outputs(dataset)
metrics = calculate_metrics(outputs, labels)
results.append({"language": language, "dataset": dataset, **metrics})
if language not in outputs_per_language:
outputs_per_language[language] = {"labels": labels, "outputs": outputs}
else:
outputs_per_language[language]["labels"] |= labels
outputs_per_language[language]["outputs"] = concatenate_datasets([
outputs_per_language[language]["outputs"], outputs
])
full_outputs = outputs if full_outputs is None else concatenate_datasets([full_outputs, outputs])
full_labels |= labels
# 🔤 Per-language evaluation
for language, data in outputs_per_language.items():
metrics = calculate_metrics(data["outputs"], data["labels"])
results.append({"language": language, "dataset": "all", **metrics})
# 🌍 Global evaluation
if full_outputs is not None:
metrics = calculate_metrics(full_outputs, full_labels)
results.append({"language": "all", "dataset": "all", **metrics})
# 💾 Save results
Dataset.from_list(results).to_json(os.path.join(outputs_path, "results.jsonl"))
``` |