longform

app.py

@@ -27,7 +27,7 @@ column_names = {
     "Voxpopuli WER": "Voxpopuli",
 }
 
-eval_queue_repo, requested_models, csv_results, multilingual_csv_path = load_all_info_from_dataset_hub()
+eval_queue_repo, requested_models, csv_results, multilingual_csv_path, longform_csv_path = load_all_info_from_dataset_hub()
 
 if not csv_results.exists():
     raise Exception(f"CSV file {csv_results} does not exist locally")
@@ -230,32 +230,57 @@ def toggle_language_expansion(language_code):
 multilingual_df = create_multilingual_dataframe()
 
 def create_longform_dataframe():
-    """Create longform dataframe with sample data for Earnings21 and MustC datasets"""
-    longform_data = []
-    
-    # Sample data with realistic WER values for longform tasks (generally higher than short-form)
-    sample_models = [
-        {"model": "openai/whisper-large-v3", "earnings21": 8.2, "mustc": 12.4, "rtfx": 2.1},
-        {"model": "openai/whisper-large-v2", "earnings21": 9.1, "mustc": 13.8, "rtfx": 1.8},
-        {"model": "nvidia/canary-1b", "earnings21": 7.5, "mustc": 11.2, "rtfx": 3.2},
-        {"model": "microsoft/speecht5_asr", "earnings21": 15.3, "mustc": 18.7, "rtfx": 1.4},
-        {"model": "facebook/wav2vec2-large-960h", "earnings21": 12.4, "mustc": 16.8, "rtfx": 0.8},
-        {"model": "assemblyai/conformer-1", "earnings21": 6.8, "mustc": 10.1, "rtfx": 2.8},
-        {"model": "speechmatics/en", "earnings21": 5.9, "mustc": 9.3, "rtfx": 3.5},
-        {"model": "revai/english", "earnings21": 6.2, "mustc": 9.8, "rtfx": 2.9},
-    ]
-    
-    for model_data in sample_models:
-        row = {
-            "Model": make_clickable_model(model_data["model"]),
-            "Average WER ⬇️": round((model_data["earnings21"] + model_data["mustc"]) / 2, 2),
-            "RTFx ⬆️️": model_data["rtfx"] if model_data["rtfx"] > 0 else "NA",
-            "Earnings21": model_data["earnings21"],
-            "MustC": model_data["mustc"]
-        }
-        longform_data.append(row)
+    """Create longform dataframe from CSV data"""
+    if longform_csv_path is None or not longform_csv_path.exists():
+        print("Longform CSV not found, creating sample data")
+        # Fallback to sample data if CSV not available
+        longform_data = []
+        sample_models = [
+            {"model": "openai/whisper-large-v3", "earnings21": 8.2, "mustc": 12.4, "rtfx": 2.1},
+            {"model": "openai/whisper-large-v2", "earnings21": 9.1, "mustc": 13.8, "rtfx": 1.8},
+            {"model": "nvidia/canary-1b", "earnings21": 7.5, "mustc": 11.2, "rtfx": 3.2},
+            {"model": "microsoft/speecht5_asr", "earnings21": 15.3, "mustc": 18.7, "rtfx": 1.4},
+        ]
+        
+        for model_data in sample_models:
+            row = {
+                "Model": make_clickable_model(model_data["model"]),
+                "Average WER ⬇️": round((model_data["earnings21"] + model_data["mustc"]) / 2, 2),
+                "RTFx ⬆️️": model_data["rtfx"] if model_data["rtfx"] > 0 else "NA",
+                "Earnings21": model_data["earnings21"],
+                "MustC": model_data["mustc"]
+            }
+            longform_data.append(row)
+        
+        longform_df = pd.DataFrame(longform_data)
+    else:
+        # Load from CSV
+        longform_raw_df = pd.read_csv(longform_csv_path)
+        longform_data = []
+        
+        for _, row_data in longform_raw_df.iterrows():
+            model_name = row_data['model']
+            
+            # Get values from CSV, similar to other tabs
+            earnings21_wer = row_data.get('earnings21_wer', -1)
+            mustc_wer = row_data.get('mustc_wer', -1)
+            rtfx_value = row_data.get('rtfx', 0)
+            
+            # Calculate average WER from available datasets
+            available_wers = [w for w in [earnings21_wer, mustc_wer] if w != -1 and w > 0]
+            avg_wer = round(np.mean(available_wers), 2) if available_wers else 0.0
+            
+            row = {
+                "Model": make_clickable_model(model_name),
+                "Average WER ⬇️": avg_wer,
+                "RTFx ⬆️️": rtfx_value if rtfx_value > 0 else "NA",
+                "Earnings21": earnings21_wer if earnings21_wer != -1 else "NA",
+                "MustC": mustc_wer if mustc_wer != -1 else "NA"
+            }
+            longform_data.append(row)
+        
+        longform_df = pd.DataFrame(longform_data)
     
-    longform_df = pd.DataFrame(longform_data)
     longform_df = longform_df.sort_values(by='Average WER ⬇️')
     return longform_df
 

init.py

@@ -6,8 +6,10 @@ from huggingface_hub import HfApi, Repository
 TOKEN_HUB = os.environ.get("TOKEN_HUB", None)
 QUEUE_REPO = os.environ.get("QUEUE_REPO", None)
 QUEUE_REPO_MULTI = os.environ.get("QUEUE_REPO_MULTI", None)
+QUEUE_REPO_LONGFORM = os.environ.get("QUEUE_REPO_LONGFORM", None)
 QUEUE_PATH = os.environ.get("QUEUE_PATH", None)
 QUEUE_PATH_MULTI = os.environ.get("QUEUE_PATH_MULTI", None)
+QUEUE_PATH_LONGFORM = os.environ.get("QUEUE_PATH_LONGFORM", None)
 
 hf_api = HfApi(
     endpoint="https://huggingface.co", 
@@ -45,8 +47,11 @@ def load_all_info_from_dataset_hub():
 
     # Load multilingual data in the same way
     multilingual_csv_results = load_multilingual_data()
+    
+    # Load longform data in the same way
+    longform_csv_results = load_longform_data()
 
-    return eval_queue_repo, requested_models, csv_results, multilingual_csv_results
+    return eval_queue_repo, requested_models, csv_results, multilingual_csv_results, longform_csv_results
 
 def load_multilingual_data():
     """Load multilingual evaluation data from CSV"""
@@ -81,6 +86,39 @@ def load_multilingual_data():
         print(f"Error loading multilingual data: {e}")
         return None
 
+def load_longform_data():
+    """Load longform evaluation data from CSV"""
+    longform_queue_path = QUEUE_PATH_LONGFORM
+    
+    try:
+        # Try to get from dedicated longform HF repo first
+        if TOKEN_HUB is not None:
+            print("Pulling longform evaluation data.")
+            try:
+                longform_repo = Repository(
+                    local_dir=longform_queue_path,
+                    clone_from=QUEUE_REPO_LONGFORM,
+                    use_auth_token=TOKEN_HUB,
+                    repo_type="dataset",
+                )
+                longform_repo.git_pull()
+                longform_csv = get_longform_csv_with_results(longform_queue_path)
+            except Exception as e:
+                print(f"Failed to pull from longform repo: {e}")
+                longform_csv = None
+        else:
+            longform_csv = None
+            
+        # Fallback to local file
+        if longform_csv is None:
+            print("Using local longform CSV file.")
+            longform_csv = get_longform_csv_with_results(".")
+            
+        return longform_csv
+    except Exception as e:
+        print(f"Error loading longform data: {e}")
+        return None
+
 
 def upload_file(requested_model_name, path_or_fileobj):
     dest_repo_file = Path(EVAL_REQUESTS_PATH) / path_or_fileobj.name
@@ -118,6 +156,18 @@ def get_multilingual_csv_with_results(directory):
         return None
     return multilingual_csv_files[0]
 
+def get_longform_csv_with_results(directory):
+    """Get longform CSV results file"""
+    directory = Path(directory)
+    longform_csv_files = list(directory.glob("longform_results_latest.csv"))
+    if len(longform_csv_files) != 1:
+        # Try local directory as fallback
+        local_longform = Path("longform_results_latest.csv")
+        if local_longform.exists():
+            return local_longform
+        return None
+    return longform_csv_files[0]
+
 
 
 def is_model_on_hub(model_name, revision="main") -> bool: