Spaces:
Running
Running
Commit
·
8b5f20b
1
Parent(s):
79051ee
Update descriptions
Browse files- app.py +11 -11
- pages/about.md +19 -8
- pages/header.md +9 -1
- pages/overall-results.md +5 -1
- pages/results-per-dataset.md +5 -1
- pages/results-per-emotion.md +5 -1
- pages/results-per-language.md +7 -1
- pages/submit.md +12 -10
- src/process_data.py +0 -2
app.py
CHANGED
@@ -19,31 +19,31 @@ metric = ["f1_macro", "accuracy", "weighted_f1"]
|
|
19 |
|
20 |
|
21 |
def app():
|
22 |
-
with gr.Blocks(theme=gr.themes.Soft(primary_hue="pink", secondary_hue="
|
23 |
-
gr.Markdown(open("pages/header.md", "r").read())
|
24 |
|
25 |
with gr.Tabs():
|
26 |
with gr.Tab("🏆 Leaderboard", elem_classes='tab-item'):
|
27 |
with gr.Tab("Overall Results", elem_classes='tab-item'):
|
28 |
-
gr.Markdown(open("pages/overall-results.md", "r").read(), elem_classes='tab-item')
|
29 |
-
overall_table = gr.Dataframe()
|
30 |
|
31 |
with gr.Tab("Results per Language", elem_classes='tab-item'):
|
32 |
-
gr.Markdown(open("pages/results-per-language.md", "r").read(), elem_classes='tab-item')
|
33 |
languages_filter = gr.CheckboxGroup(choices=languages, label="Select columns", value=languages)
|
34 |
select_lang_metric = gr.Radio(metric, value='f1_macro', label="Select metric")
|
35 |
-
lang_table = gr.Dataframe()
|
36 |
|
37 |
with gr.Tab("Results per Dataset", elem_classes='tab-item'):
|
38 |
-
gr.Markdown(open("pages/results-per-dataset.md", "r").read(), elem_classes='tab-item')
|
39 |
dataset_filter = gr.CheckboxGroup(choices=datasets, label="Select columns", value=datasets)
|
40 |
select_ds_metric = gr.Radio(metric, value='f1_macro', label="Select metric")
|
41 |
-
dataset_table = gr.Dataframe()
|
42 |
|
43 |
with gr.Tab("Results per Emotion", elem_classes='tab-item'):
|
44 |
-
gr.Markdown(open("pages/results-per-emotion.md", "r").read(), elem_classes='tab-item')
|
45 |
emo_filter = gr.CheckboxGroup(choices=emotions, label="Select columns", value=emotions)
|
46 |
-
emotion_table = gr.Dataframe()
|
47 |
|
48 |
df_state = gr.State()
|
49 |
|
@@ -88,7 +88,7 @@ def app():
|
|
88 |
with gr.Tab("🔢 Evaluate your model", elem_classes='tab-item'):
|
89 |
gr.Markdown(open("pages/evaluate.md", "r").read(), elem_classes='tab-item')
|
90 |
|
91 |
-
with gr.Tab("
|
92 |
gr.Markdown(open("pages/submit.md", "r").read(), elem_classes='tab-item')
|
93 |
|
94 |
with gr.Column():
|
|
|
19 |
|
20 |
|
21 |
def app():
|
22 |
+
with gr.Blocks(theme=gr.themes.Soft(primary_hue="pink", secondary_hue="purple")) as demo:
|
23 |
+
gr.Markdown(open("pages/header.md", "r").read(), container=True)
|
24 |
|
25 |
with gr.Tabs():
|
26 |
with gr.Tab("🏆 Leaderboard", elem_classes='tab-item'):
|
27 |
with gr.Tab("Overall Results", elem_classes='tab-item'):
|
28 |
+
gr.Markdown(open("pages/overall-results.md", "r").read(), elem_classes='tab-item', container=True)
|
29 |
+
overall_table = gr.Dataframe(show_row_numbers=True, pinned_columns=2)
|
30 |
|
31 |
with gr.Tab("Results per Language", elem_classes='tab-item'):
|
32 |
+
gr.Markdown(open("pages/results-per-language.md", "r").read(), elem_classes='tab-item', container=True)
|
33 |
languages_filter = gr.CheckboxGroup(choices=languages, label="Select columns", value=languages)
|
34 |
select_lang_metric = gr.Radio(metric, value='f1_macro', label="Select metric")
|
35 |
+
lang_table = gr.Dataframe(show_row_numbers=True, pinned_columns=2)
|
36 |
|
37 |
with gr.Tab("Results per Dataset", elem_classes='tab-item'):
|
38 |
+
gr.Markdown(open("pages/results-per-dataset.md", "r").read(), elem_classes='tab-item', container=True)
|
39 |
dataset_filter = gr.CheckboxGroup(choices=datasets, label="Select columns", value=datasets)
|
40 |
select_ds_metric = gr.Radio(metric, value='f1_macro', label="Select metric")
|
41 |
+
dataset_table = gr.Dataframe(show_row_numbers=True, pinned_columns=2)
|
42 |
|
43 |
with gr.Tab("Results per Emotion", elem_classes='tab-item'):
|
44 |
+
gr.Markdown(open("pages/results-per-emotion.md", "r").read(), elem_classes='tab-item', container=True)
|
45 |
emo_filter = gr.CheckboxGroup(choices=emotions, label="Select columns", value=emotions)
|
46 |
+
emotion_table = gr.Dataframe(show_row_numbers=True, pinned_columns=2)
|
47 |
|
48 |
df_state = gr.State()
|
49 |
|
|
|
88 |
with gr.Tab("🔢 Evaluate your model", elem_classes='tab-item'):
|
89 |
gr.Markdown(open("pages/evaluate.md", "r").read(), elem_classes='tab-item')
|
90 |
|
91 |
+
with gr.Tab("📬 Submit here!", elem_classes='tab-item'):
|
92 |
gr.Markdown(open("pages/submit.md", "r").read(), elem_classes='tab-item')
|
93 |
|
94 |
with gr.Column():
|
pages/about.md
CHANGED
@@ -1,14 +1,25 @@
|
|
1 |
# 📝 About
|
2 |
|
3 |
-
|
4 |
-
|
|
|
5 |
|
6 |
-
|
7 |
-
|
|
|
|
|
8 |
|
9 |
-
|
10 |
-
|
|
|
|
|
|
|
|
|
11 |
|
12 |
-
|
|
|
13 |
|
14 |
-
|
|
|
|
|
|
|
|
1 |
# 📝 About
|
2 |
|
3 |
+
**CAMEO** (**C**ollection of Multilingu**a**l E**m**otional Sp**e**ech C**o**rpora) is a benchmark
|
4 |
+
dataset designed to support research in **Speech Emotion Recognition (SER)** — especially in
|
5 |
+
multilingual and cross-lingual settings.
|
6 |
|
7 |
+
The collection brings together **13 emotional speech datasets** covering **8 languages**, including
|
8 |
+
English, German, Spanish, French, Serbian, and more. In total, it contains **41,265 audio samples**,
|
9 |
+
with each sample annotated for **emotion**, and in most cases, also for **speaker ID**, **gender**,
|
10 |
+
and **age**.
|
11 |
|
12 |
+
Here are a few quick facts about the dataset:
|
13 |
+
- Over **33%** of the samples are in English.
|
14 |
+
- **17 distinct emotional states** are represented across datasets.
|
15 |
+
- **93.5%** of samples fall under the **seven primary emotions**: *neutral*, *anger*, *sadness*,
|
16 |
+
*surprise*, *happiness*, *disgust*, and *fear*.
|
17 |
+
- **Gender annotations** are available for over **92%** of samples.
|
18 |
|
19 |
+
All datasets included in **CAMEO** are openly available. We've made the full collection accessible
|
20 |
+
on Hugging Face, along with metadata, tools, and a leaderboard for evaluation.
|
21 |
|
22 |
+
🔗 [View the **CAMEO** Dataset on Hugging Face](https://huggingface.co/datasets/amu-cai/CAMEO)
|
23 |
+
|
24 |
+
Whether you're building SER models or exploring emotion understanding across languages, **CAMEO**
|
25 |
+
is here to support your research.
|
pages/header.md
CHANGED
@@ -1 +1,9 @@
|
|
1 |
-
# Open Speech Emotion Recognition Leaderboard
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Open Speech Emotion Recognition Leaderboard
|
2 |
+
|
3 |
+
Welcome to the **Open SER Leaderboard** — part of the [**CAMEO**](https://huggingface.co/datasets/amu-cai/CAMEO) project!
|
4 |
+
|
5 |
+
This leaderboard tracks how well different models recognize emotions in speech across multiple languages.
|
6 |
+
|
7 |
+
Everything is open, transparent, and reproducible — you're invited to explore, evaluate, and contribute.
|
8 |
+
|
9 |
+
<br><br>
|
pages/overall-results.md
CHANGED
@@ -1 +1,5 @@
|
|
1 |
-
|
|
|
|
|
|
|
|
|
|
1 |
+
This tab shows how different models perform across the entire **CAMEO** collection. You’ll find
|
2 |
+
**macro F1**, **weighted F1**, and **accuracy** scores for each model, tested at different
|
3 |
+
temperature settings.
|
4 |
+
|
5 |
+
It's a great place to get a quick overview of how models compare on the full dataset.
|
pages/results-per-dataset.md
CHANGED
@@ -1 +1,5 @@
|
|
1 |
-
|
|
|
|
|
|
|
|
|
|
1 |
+
This tab breaks down results by individual datasets included in the **CAMEO** collection. You can
|
2 |
+
choose which datasets to view and which metric to focus on.
|
3 |
+
|
4 |
+
It’s helpful for spotting differences in performance, potential data overlap, or just understanding
|
5 |
+
how models behave on different kinds of emotional speech.
|
pages/results-per-emotion.md
CHANGED
@@ -1 +1,5 @@
|
|
1 |
-
|
|
|
|
|
|
|
|
|
|
1 |
+
Which emotions are easier for models to recognize — and which ones still trip them up? This view
|
2 |
+
shows how models perform on specific emotional states.
|
3 |
+
|
4 |
+
Pick the emotions and metric you’re interested in, and see which models handle them best. It's a
|
5 |
+
great tool for digging deeper into model behavior.
|
pages/results-per-language.md
CHANGED
@@ -1 +1,7 @@
|
|
1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
Curious how models handle different languages? This view lets you compare performance across
|
2 |
+
languages like English, French, German, and more. Use the checkboxes to pick which languages you
|
3 |
+
want to see, and switch between metrics like **macro F1**, **weighted F1**, or **accuracy** using
|
4 |
+
the radio buttons.
|
5 |
+
|
6 |
+
This is especially useful if you’re working on multilingual models or looking to improve
|
7 |
+
performance in a specific language.
|
pages/submit.md
CHANGED
@@ -1,15 +1,17 @@
|
|
1 |
-
#
|
2 |
|
3 |
-
|
4 |
|
5 |
-
|
6 |
|
7 |
-
|
|
|
|
|
|
|
|
|
8 |
|
9 |
-
|
|
|
|
|
10 |
|
11 |
-
|
12 |
-
|
13 |
-
🏆 **Step 3:** Once evaluated, your model will join the leaderboard.
|
14 |
-
|
15 |
-
Thanks for sharing your work with us and making this project even better!
|
|
|
1 |
+
# 📬 Submit Here!
|
2 |
|
3 |
+
Want your model to appear on the leaderboard?
|
4 |
|
5 |
+
Send us an email at [[email protected]](mailto:[email protected]?subject=CAMEO%20Leaderboard%20Submission) with the subject line **"CAMEO Leaderboard Submission"**.
|
6 |
|
7 |
+
Please include:
|
8 |
+
- Your model's name and a short description.
|
9 |
+
- The temperature setting you used.
|
10 |
+
- A JSONL file with your predictions.
|
11 |
+
- Any other details you'd like to share.
|
12 |
|
13 |
+
If you **don’t have access to the resources** needed to run the evaluation yourself, no
|
14 |
+
problem — just send us a **link to the model** (e.g., a Hugging Face model page), and we’ll do our
|
15 |
+
best to run the evaluation for you.
|
16 |
|
17 |
+
We’ll review your submission and add your results to the leaderboard!
|
|
|
|
|
|
|
|
src/process_data.py
CHANGED
@@ -6,7 +6,6 @@ def overall_leaderboard(df: pd.DataFrame, sort_column: str = "f1_macro"):
|
|
6 |
df = df[df["language"] == "All"]
|
7 |
df = df[["model", "temperature", "f1_macro", "weighted_f1", "accuracy"]]
|
8 |
df = df.sort_values(by=sort_column, ascending=False)
|
9 |
-
df.insert(0, "Rank", range(1, len(df) + 1))
|
10 |
|
11 |
df["temperature"] = df["temperature"].round(1)
|
12 |
|
@@ -67,6 +66,5 @@ def leaderboard_per_group(lang_dict, use_cols, metric: str = "f1_macro"):
|
|
67 |
df = df[["model", "temperature"] + sorted(use_cols)]
|
68 |
if "All" in use_cols:
|
69 |
df = df.sort_values(by="All", ascending=False)
|
70 |
-
df.insert(0, "Rank", range(1, len(df) + 1))
|
71 |
|
72 |
return df
|
|
|
6 |
df = df[df["language"] == "All"]
|
7 |
df = df[["model", "temperature", "f1_macro", "weighted_f1", "accuracy"]]
|
8 |
df = df.sort_values(by=sort_column, ascending=False)
|
|
|
9 |
|
10 |
df["temperature"] = df["temperature"].round(1)
|
11 |
|
|
|
66 |
df = df[["model", "temperature"] + sorted(use_cols)]
|
67 |
if "All" in use_cols:
|
68 |
df = df.sort_values(by="All", ascending=False)
|
|
|
69 |
|
70 |
return df
|