add multilingual leaderboad (#41)

- add multilingual leaderboard (95a0502ab442e9c6419383be1760c4067a9af12c)

app.py

@@ -1,13 +1,18 @@
 import gradio as gr
 import pandas as pd
 import json
-from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS, LEADERBOARD_CSS
+from constants import BANNER, INTRODUCTION_TEXT, CITATION_TEXT, METRICS_TAB_TEXT, DIR_OUTPUT_REQUESTS, LEADERBOARD_CSS, EU_LANGUAGES, MULTILINGUAL_TAB_TEXT
 from init import is_model_on_hub, upload_file, load_all_info_from_dataset_hub
-from utils_display import AutoEvalColumn, fields, make_clickable_model, styled_error, styled_message
+from utils_display import AutoEvalColumn, MultilingualColumn, fields, make_clickable_model, styled_error, styled_message
+import numpy as np
 from datetime import datetime, timezone
 
 LAST_UPDATED = "Apr 8th 2025"
 
+# Global variable to store detailed benchmark data
+benchmark_details = {}
+expanded_languages = set()  # Track which languages are expanded
+
 column_names = {
     "MODEL": "Model",
     "Avg. WER": "Average WER ⬇️",    
@@ -22,7 +27,7 @@ column_names = {
     "Voxpopuli WER": "Voxpopuli",
 }
 
-eval_queue_repo, requested_models, csv_results = load_all_info_from_dataset_hub()
+eval_queue_repo, requested_models, csv_results, multilingual_csv_path = load_all_info_from_dataset_hub()
 
 if not csv_results.exists():
     raise Exception(f"CSV file {csv_results} does not exist locally")
@@ -49,6 +54,177 @@ original_df.sort_values(by='Average WER ⬇️', inplace=True)
 COLS = [c.name for c in fields(AutoEvalColumn)]
 TYPES = [c.type for c in fields(AutoEvalColumn)]
 
+# Multilingual columns (dynamic based on expansion state)
+MULTILINGUAL_COLS = [c.name for c in fields(MultilingualColumn)]
+
+def create_multilingual_dataframe():
+    """Create multilingual dataframe with CoVoST, MLS, and FLEURS benchmark data"""
+    global benchmark_details, expanded_languages
+    
+    if multilingual_csv_path is None or not multilingual_csv_path.exists():
+        raise Exception("Multilingual CSV file not found")
+    
+    # Load CSV data
+    multilingual_raw_df = pd.read_csv(multilingual_csv_path)
+    
+    # Store detailed benchmark data for click functionality
+    benchmark_details = {}
+    
+    multilingual_data = []
+    for _, row_data in multilingual_raw_df.iterrows():
+        model_name = row_data['model']
+        model_details = {}
+        row = {"Model": make_clickable_model(model_name)}
+        
+        # Process data for each language and collect all individual datapoints
+        all_datapoints = []  # Collect all individual dataset scores across all languages
+        
+        for lang_code, lang_info in EU_LANGUAGES.items():
+            # Get individual benchmark scores from CSV, using None for missing values
+            # Special cases: de doesn't have MLS, pt doesn't have CoVoST
+            if lang_code == "pt":
+                covost_score = None  # pt doesn't have CoVoST data
+            else:
+                covost_score = row_data.get(f"{lang_code}_covost", None)
+            
+            if lang_code == "de":
+                mls_score = None  # de doesn't have MLS data
+            else:
+                mls_score = row_data.get(f"{lang_code}_mls", None)
+            
+            fleurs_score = row_data.get(f"{lang_code}_fleurs", None)
+            
+            # Convert string zeros or empty values to None
+            for score_name, score_val in [("covost", covost_score), ("mls", mls_score), ("fleurs", fleurs_score)]:
+                if score_val is not None and (score_val == 0.0 or score_val == "" or str(score_val).strip() == "0" or str(score_val).strip() == ""):
+                    if score_name == "covost":
+                        covost_score = None
+                    elif score_name == "mls":
+                        mls_score = None
+                    elif score_name == "fleurs":
+                        fleurs_score = None
+            
+            # Add individual datapoints to the global list
+            if covost_score is not None and covost_score > 0:
+                all_datapoints.append(covost_score)
+            if mls_score is not None and mls_score > 0:
+                all_datapoints.append(mls_score)
+            if fleurs_score is not None and fleurs_score > 0:
+                all_datapoints.append(fleurs_score)
+            
+            # Calculate average only from available scores for this language (for display)
+            available_scores = [s for s in [covost_score, mls_score, fleurs_score] if s is not None and s > 0]
+            if available_scores:
+                avg_score = round(sum(available_scores) / len(available_scores), 2)
+            else:
+                avg_score = None
+            
+            # Store individual scores for detailed view (only store existing datasets)
+            lang_data = {"average": avg_score if avg_score is not None else "NA"}
+            
+            # Only store datasets that exist for this language
+            if lang_code != "pt" and covost_score is not None:  # pt doesn't have CoVoST
+                lang_data["CoVoST"] = covost_score
+            if lang_code != "de" and mls_score is not None:  # de doesn't have MLS
+                lang_data["MLS"] = mls_score
+            if fleurs_score is not None:  # All languages have FLEURS
+                lang_data["FLEURS"] = fleurs_score
+                
+            model_details[lang_code] = lang_data
+        
+        # Calculate overall multilingual average from all individual datapoints
+        if all_datapoints:
+            row["Average WER ⬇️"] = round(np.mean(all_datapoints), 2)
+        else:
+            row["Average WER ⬇️"] = 0.0
+        
+        # Add RTFx from the CSV (it should be a single value per model)
+        rtfx_value = row_data.get("rtfx", row_data.get("RTFx", 0.0))
+        # Convert 0 or -1 values to "NA" like in the English leaderboard
+        if rtfx_value == 0.0 or rtfx_value == -1 or rtfx_value == 0 or rtfx_value == "0" or rtfx_value == "0.0":
+            row["RTFx ⬆️️"] = "NA"
+        else:
+            row["RTFx ⬆️️"] = rtfx_value
+        
+        # Add language columns based on expansion state
+        for lang_code, lang_info in EU_LANGUAGES.items():
+            lang_col_name = f"{lang_info['flag']} {lang_info['name']}"
+            model_data = model_details[lang_code]
+            
+            if lang_code in expanded_languages:
+                # Show average column AND detailed columns
+                row[f"{lang_col_name} Avg"] = model_data["average"]
+                
+                # Only show columns for datasets that actually exist in the data
+                if "CoVoST" in model_data:
+                    row[f"{lang_col_name} CoVoST"] = model_data["CoVoST"]
+                if "MLS" in model_data:
+                    row[f"{lang_col_name} MLS"] = model_data["MLS"]
+                if "FLEURS" in model_data:
+                    row[f"{lang_col_name} FLEURS"] = model_data["FLEURS"]
+            else:
+                # Show only average column
+                row[lang_col_name] = model_data["average"]
+        
+        # Store model details for click functionality
+        benchmark_details[model_name] = model_details
+        multilingual_data.append(row)
+    
+    multilingual_df = pd.DataFrame(multilingual_data)
+    multilingual_df = multilingual_df.sort_values(by='Average WER ⬇️')
+    return multilingual_df
+
+def get_multilingual_datatypes(df):
+    """Generate appropriate datatypes for multilingual dataframe columns"""
+    datatypes = []
+    for col in df.columns:
+        if col == "Model":
+            datatypes.append("markdown")  # This allows HTML rendering
+        else:
+            datatypes.append("number")
+    return datatypes
+
+def get_language_details(model, language_code):
+    """Get detailed breakdown for a specific model and language"""
+    global benchmark_details
+    
+    if model not in benchmark_details or language_code not in benchmark_details[model]:
+        return None
+    
+    language_info = EU_LANGUAGES.get(language_code, {})
+    language_name = language_info.get("name", "Unknown")
+    model_data = benchmark_details[model][language_code]
+    
+    details = {
+        "Language": f"{language_info.get('flag', '')} {language_name}",
+        "Model": model,
+        "CoVoST WER": model_data["CoVoST"],
+        "MLS WER": model_data["MLS"],
+        "FLEURS WER": model_data["FLEURS"],
+        "Average WER": model_data["average"]
+    }
+    
+    return details
+
+def toggle_language_expansion(language_code):
+    """Toggle expansion of language columns when button is clicked"""
+    global expanded_languages
+    
+    # Toggle expansion state
+    if language_code in expanded_languages:
+        expanded_languages.remove(language_code)
+    else:
+        expanded_languages.add(language_code)
+    
+    # Recreate dataframe with new expansion state
+    updated_df = create_multilingual_dataframe()
+    updated_datatypes = get_multilingual_datatypes(updated_df)
+    
+    return gr.update(value=updated_df, datatype=updated_datatypes)
+
+# Initialize multilingual dataframe
+multilingual_df = create_multilingual_dataframe()
+
 
 def request_model(model_text, chbcoco2017):
     
@@ -138,10 +314,59 @@ with gr.Blocks(css=LEADERBOARD_CSS) as demo:
                 outputs=leaderboard_table
             )
 
-        with gr.TabItem("📈 Metrics", elem_id="od-benchmark-tab-table", id=2):
+        with gr.TabItem("🌍 Multilingual", elem_id="multilingual-benchmark-tab-table", id=1):
+            gr.Markdown(MULTILINGUAL_TAB_TEXT, elem_classes="markdown-text")
+            
+            # Language toggle buttons
+            gr.Markdown("Click on a language button to show/hide detailed benchmark scores (CoVoST, MLS, FLEURS):")
+            
+            language_buttons = {}
+            lang_codes = list(EU_LANGUAGES.keys())
+            
+            # First row of buttons (5 languages)
+            with gr.Row():
+                for lang_code in lang_codes[:5]:
+                    lang_info = EU_LANGUAGES[lang_code]
+                    button_label = f"{lang_info['flag']} {lang_info['name']}"
+                    language_buttons[lang_code] = gr.Button(
+                        button_label, 
+                        variant="secondary",
+                        size="sm"
+                    )
+            
+            # Second row of buttons (remaining 5 languages)
+            with gr.Row():
+                for lang_code in lang_codes[5:]:
+                    lang_info = EU_LANGUAGES[lang_code]
+                    button_label = f"{lang_info['flag']} {lang_info['name']}"
+                    language_buttons[lang_code] = gr.Button(
+                        button_label, 
+                        variant="secondary",
+                        size="sm"
+                    )
+            
+            multilingual_table = gr.components.Dataframe(
+                value=multilingual_df,
+                datatype=get_multilingual_datatypes(multilingual_df),
+                elem_id="multilingual-table",
+                interactive=False,
+                visible=True,
+            )
+            
+            # Connect buttons to toggle language expansion
+            for lang_code, button in language_buttons.items():
+                def create_toggle_func(code):
+                    return lambda: toggle_language_expansion(code)
+                
+                button.click(
+                    create_toggle_func(lang_code),
+                    outputs=[multilingual_table]
+                )
+
+        with gr.TabItem("📈 Metrics", elem_id="od-benchmark-tab-table", id=3):
             gr.Markdown(METRICS_TAB_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("✉️✨ Request a model here!", elem_id="od-benchmark-tab-table", id=3):
+        with gr.TabItem("✉️✨ Request a model here!", elem_id="od-benchmark-tab-table", id=4):
             with gr.Column():
                 gr.Markdown("# ✉️✨ Request results for a new model here!", elem_classes="markdown-text")
             with gr.Column():
@@ -156,7 +381,7 @@ with gr.Blocks(css=LEADERBOARD_CSS) as demo:
                                       [model_name_textbox, chb_coco2017], 
                                       mdw_submission_result)
         # add an about section
-        with gr.TabItem("🤗 About", elem_id="od-benchmark-tab-table", id=4):
+        with gr.TabItem("🤗 About", elem_id="od-benchmark-tab-table", id=5):
             gr.Markdown("## About", elem_classes="markdown-text")
 
     gr.Markdown(f"Last updated on **{LAST_UPDATED}**", elem_classes="markdown-text")

constants.py

@@ -17,7 +17,7 @@ INTRODUCTION_TEXT = "📐 The 🤗 Open ASR Leaderboard ranks and evaluates spee
     on the Hugging Face Hub. \
     \nWe report the Average [WER](https://huggingface.co/spaces/evaluate-metric/wer) (⬇️ lower the better) and [RTFx](https://github.com/NVIDIA/DeepLearningExamples/blob/master/Kaldi/SpeechRecognition/README.md#metrics) (⬆️ higher the better). Models are ranked based on their Average WER, from lowest to highest. Check the 📈 Metrics tab to understand how the models are evaluated. \
     \nIf you want results for a model that is not listed here, you can submit a request for it to be included ✉️✨. \
-    \nThe leaderboard currently focuses on English speech recognition, and will be expanded to multilingual evaluation in later versions."
+    \nThe leaderboard includes both English ASR evaluation and multilingual benchmarks across the top European languages."
 
 CITATION_TEXT = """@misc{open-asr-leaderboard,
 	title        = {Open Automatic Speech Recognition Leaderboard},
@@ -114,8 +114,38 @@ are ranked based on their average WER scores, from lowest to highest.
 For more details on the individual datasets and how models are evaluated to give the ESB score, refer to the [ESB paper](https://arxiv.org/abs/2210.13352).
 """
 
+# Multilingual benchmark definitions
+EU_LANGUAGES = {
+    "de": {"name": "German", "flag": "🇩🇪", "datasets": ["mls", "fleurs", "covost"]},
+    "fr": {"name": "French", "flag": "🇫🇷", "datasets": ["mls", "fleurs", "covost"]},
+    "it": {"name": "Italian", "flag": "🇮🇹", "datasets": ["mls", "fleurs", "covost"]},
+    "es": {"name": "Spanish", "flag": "🇪🇸", "datasets": ["mls", "fleurs", "covost"]},
+    "pt": {"name": "Portuguese", "flag": "🇵🇹", "datasets": ["mls", "fleurs", "covost"]}
+}
+
+MULTILINGUAL_TAB_TEXT = """
+## 🌍 Multilingual ASR Evaluation
+
+"""
+
 LEADERBOARD_CSS = """
 #leaderboard-table th .header-content {
     white-space: nowrap;
 }
+
+#multilingual-table th .header-content {
+    white-space: nowrap;
+}
+
+#multilingual-table th:hover {
+    background-color: var(--table-row-focus);
+}
+
+.language-detail-modal {
+    background: var(--background-fill-primary);
+    border: 1px solid var(--border-color-primary);
+    border-radius: 8px;
+    padding: 1rem;
+    margin: 1rem 0;
+}
 """

init.py

@@ -5,7 +5,9 @@ from huggingface_hub import HfApi, Repository
 
 TOKEN_HUB = os.environ.get("TOKEN_HUB", None)
 QUEUE_REPO = os.environ.get("QUEUE_REPO", None)
+QUEUE_REPO_MULTI = os.environ.get("QUEUE_REPO_MULTI", None)
 QUEUE_PATH = os.environ.get("QUEUE_PATH", None)
+QUEUE_PATH_MULTI = os.environ.get("QUEUE_PATH_MULTI", None)
 
 hf_api = HfApi(
     endpoint="https://huggingface.co", 
@@ -41,7 +43,43 @@ def load_all_info_from_dataset_hub():
     if not passed:
         raise ValueError("No Hugging Face token provided. Skipping evaluation requests and results.")
 
-    return eval_queue_repo, requested_models, csv_results
+    # Load multilingual data in the same way
+    multilingual_csv_results = load_multilingual_data()
+
+    return eval_queue_repo, requested_models, csv_results, multilingual_csv_results
+
+def load_multilingual_data():
+    """Load multilingual evaluation data from CSV"""
+    multilingual_queue_path = QUEUE_PATH_MULTI
+    
+    try:
+        # Try to get from dedicated multilingual HF repo first
+        if TOKEN_HUB is not None:
+            print("Pulling multilingual evaluation data.")
+            try:
+                multilingual_repo = Repository(
+                    local_dir=multilingual_queue_path,
+                    clone_from=QUEUE_REPO_MULTI,
+                    use_auth_token=TOKEN_HUB,
+                    repo_type="dataset",
+                )
+                multilingual_repo.git_pull()
+                multilingual_csv = get_multilingual_csv_with_results(multilingual_queue_path)
+            except Exception as e:
+                print(f"Failed to pull from multilingual repo: {e}")
+                multilingual_csv = None
+        else:
+            multilingual_csv = None
+            
+        # Fallback to local file
+        if multilingual_csv is None:
+            print("Using local multilingual CSV file.")
+            multilingual_csv = get_multilingual_csv_with_results(".")
+            
+        return multilingual_csv
+    except Exception as e:
+        print(f"Error loading multilingual data: {e}")
+        return None
 
 
 def upload_file(requested_model_name, path_or_fileobj):
@@ -68,6 +106,18 @@ def get_csv_with_results(directory):
         return None
     return latest[0]
 
+def get_multilingual_csv_with_results(directory):
+    """Get multilingual CSV results file"""
+    directory = Path(directory)
+    multilingual_csv_files = list(directory.glob("multilingual_results_latest.csv"))
+    if len(multilingual_csv_files) != 1:
+        # Try local directory as fallback
+        local_multilingual = Path("multilingual_results_latest.csv")
+        if local_multilingual.exists():
+            return local_multilingual
+        return None
+    return multilingual_csv_files[0]
+
 
 
 def is_model_on_hub(model_name, revision="main") -> bool:

utils_display.py

@@ -24,6 +24,22 @@ class AutoEvalColumn: # Auto evals column
     tl_wer = ColumnContent("Tedlium", "number")
     vp_wer = ColumnContent("Voxpopuli", "number")
 
+@dataclass(frozen=True)
+class MultilingualColumn: # Multilingual benchmark columns
+    model = ColumnContent("Model", "markdown")
+    avg_multilingual = ColumnContent("Average WER ⬇️", "number")
+    rtf = ColumnContent("RTFx ⬆️️", "number")
+    de_avg = ColumnContent("🇩🇪 German", "number")
+    fr_avg = ColumnContent("🇫🇷 French", "number")
+    es_avg = ColumnContent("🇪🇸 Spanish", "number")
+    it_avg = ColumnContent("🇮🇹 Italian", "number")
+    nl_avg = ColumnContent("🇳🇱 Dutch", "number")
+    pl_avg = ColumnContent("🇵🇱 Polish", "number")
+    pt_avg = ColumnContent("🇵🇹 Portuguese", "number")
+    cs_avg = ColumnContent("🇨🇿 Czech", "number")
+    ro_avg = ColumnContent("🇷🇴 Romanian", "number")
+    hu_avg = ColumnContent("🇭🇺 Hungarian", "number")
+
 
 def make_clickable_model(model_name):
     model_name_list = model_name.split("/")