static json files

app.py

@@ -10,6 +10,10 @@ logging.basicConfig(level=logging.INFO)
 BACKEND_URL = "http://127.0.0.1:8000"
 LAST_UPDATED = "Dec 4th 2024"
 
+def load_static_leaderboard():
+    # read fake_queue/leaderboard.json
+    return pd.read_json(Path("fake_queue/leaderboard.json"))
+
 def load_leaderboard_data():
     try:
         response = requests.get(f"{BACKEND_URL}/leaderboard")
@@ -94,10 +98,10 @@ with gr.Blocks(css="""
     
     with gr.Tabs():
         with gr.TabItem("🏆 Leaderboard"):
-            leaderboard_html = gr.HTML(create_html_table(format_leaderboard_df(load_leaderboard_data())))
+            leaderboard_html = gr.HTML(create_html_table(format_leaderboard_df(load_static_leaderboard())))
             refresh_btn = gr.Button("🔄 Refresh")
             refresh_btn.click(
-                lambda: create_html_table(format_leaderboard_df(load_leaderboard_data())),
+                lambda: create_html_table(format_leaderboard_df(load_static_leaderboard())),
                 outputs=leaderboard_html
             )
         

fake_queue/leaderboard.json

@@ -0,0 +1,62 @@
+[
+    {
+        "submission_id": "8e6a3a00-59fa-4a24-861d-a132a8212658",
+        "submission_name": "facebook espeak",
+        "model": "facebook/wav2vec2-lv-60-espeak-cv-ft",
+        "average_per": 0.33667301260691423,
+        "average_pwed": 0.1276725657099669,
+        "subset": "test",
+        "github_url": "https://github.com/facebookresearch/fairseq/blob/main/examples/wav2vec/README.md",
+        "submission_date": "2024-12-05T07:32:06.850230"
+    },
+    {
+        "submission_id": "70aceb68-ad86-4a83-9998-08adb27b4d5c",
+        "submission_name": "english phoneme model",
+        "model": "KoelLabs/xlsr-timit-b0",
+        "average_per": 0.12572285528714347,
+        "average_pwed": 0.06476636812791145,
+        "subset": "test",
+        "github_url": "https://github.com/KoelLabs/",
+        "submission_date": "2024-12-05T08:25:24.982477"
+    },
+    {
+        "submission_id": "80b57299-b3ab-4caf-ac4a-898c8398046e",
+        "submission_name": "speech 31 model",
+        "model": "speech31/wav2vec2-large-TIMIT-IPA",
+        "average_per": 0.4415425496841929,
+        "average_pwed": 0.18625930002594002,
+        "subset": "test",
+        "github_url": "https://huggingface.co/speech31/wav2vec2-large-TIMIT-IPA2",
+        "submission_date": "2024-12-05T09:36:14.570315"
+    },
+    {
+        "submission_id": "0cbcab0a-bd07-421f-82a0-480c9507a214",
+        "submission_name": "jubiliano model wav2vec2",
+        "model": "Jubliano/wav2vec2-large-xls-r-300m-ipa-INTERNATIONAL1.5",
+        "average_per": 0.6318471187460027,
+        "average_pwed": 0.222932144739126,
+        "subset": "test",
+        "github_url": "https://huggingface.co/Jubliano/wav2vec2-large-xls-r-300m-ipa-INTERNATIONAL1.5WithoutSpaces/tree/d5312009d8e620b183c334dfdd9ffc6b4f06f8c1",
+        "submission_date": "2024-12-05T10:17:21.334530"
+    },
+    {
+        "submission_id": "0fc29c54-3db2-46b6-aeee-c96484306751",
+        "submission_name": "xlsr 53 model",
+        "model": "facebook/wav2vec2-xlsr-53-espeak-cv-ft",
+        "average_per": 0.348845592557092,
+        "average_pwed": 0.1386742019529415,
+        "subset": "test",
+        "github_url": "https://github.com/facebookresearch/fairseq/blob/main/examples/wav2vec/README.md",
+        "submission_date": "2024-12-05T10:34:26.157054"
+    },
+    {
+        "submission_id": "a23026ec-acac-4481-9761-f9368b4b94f1",
+        "submission_name": "ginic model wav2vec2 finetuned on buckeye",
+        "model": "ginic/hyperparam_tuning_1_wav2vec2-large-xlsr-buckeye-ipa",
+        "average_per": 0.2766466385175833,
+        "average_pwed": 0.10410683992600853,
+        "subset": "test",
+        "github_url": "https://huggingface.co/ginic/vary_individuals_old_only_1_wav2vec2-large-xlsr-buckeye-ipa",
+        "submission_date": "2024-12-05T11:06:07.984825"
+    }
+]

fake_queue/results.json

@@ -0,0 +1,278 @@
+[
+    {
+        "task_id": "721b4c64-a825-42d3-bb0a-bdff9ee1ed0f",
+        "model": "facebook/wav2vec2-lv-60-espeak-cv-ft",
+        "subset": "test",
+        "num_files": 1680,
+        "average_per": 0.33667301260691423,
+        "average_pwed": 0.1276725657099669,
+        "detailed_results": [
+            {
+                "file": "data/TEST/DR1/FAKS0/SA1.WAV",
+                "ground_truth": "ʃihædjɹdɑɹksuɾɪŋgɹisiwɑʃwɑɾɹʔɔljiɹ",
+                "prediction": "ʃiːhædjɚdɑːɹksuːɾɪnɡɹiːsiwɑːʃwɑːɾɚɹɑːljiː",
+                "per": 0.3939393939393939,
+                "pwed": 0.13888888888888887
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SA2.WAV",
+                "ground_truth": "oʊnæsmitikɛɹiinɔɪliɹæglaɪkðæt",
+                "prediction": "doʊntæskmiːtəkæɹiɐnoɪliɹæɡlaɪkðæt",
+                "per": 0.32142857142857145,
+                "pwed": 0.13541666666666666
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI1573.WAV",
+                "ground_truth": "hɪzkæpinwəsθɪnænhægɹdinɪzbjuɾuflbutswɹwɔɹninʃæbi",
+                "prediction": "hɪzkæptənwʌzθɪnændhæɡɚdændhɪzbjuːɾɪfəlbuːtswɜːwɔːɹnændʃæbi",
+                "per": 0.3617021276595745,
+                "pwed": 0.13915094339622644
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI2203.WAV",
+                "ground_truth": "ðiɹizənzfɹðɪsdaɪvsimdfuliʃnaʊ",
+                "prediction": "ðəɹiːzənzfɜːðɪsdaɪvsiːmdfuːlɪʃnaʊ",
+                "per": 0.20689655172413793,
+                "pwed": 0.022988505747126433
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI943.WAV",
+                "ground_truth": "ɹdʌkʃinmeɪfɔlfɑɹbəloʊəkspikeɪʃnts",
+                "prediction": "pɹədʌkʃənmeɪfɔːlfɑːɹbᵻloʊɛkspɛkteɪʃənz",
+                "per": 0.36363636363636365,
+                "pwed": 0.1392857142857143
+            }
+        ],
+        "timestamp": "2024-12-05T07:32:06.849017"
+    },
+    {
+        "task_id": "d6fe0956-b5b4-4105-835e-8dee1872ee4d",
+        "model": "KoelLabs/xlsr-timit-b0",
+        "subset": "test",
+        "num_files": 1680,
+        "average_per": 0.12572285528714347,
+        "average_pwed": 0.06476636812791145,
+        "detailed_results": [
+            {
+                "file": "data/TEST/DR1/FAKS0/SA1.WAV",
+                "ground_truth": "ʃihædjɹdɑɹksuɾɪŋgɹisiwɑʃwɑɾɹʔɔljiɹ",
+                "prediction": "ʃihædjɹdɑɹksuɾɪnɡɹisiwɑʃwɔɾɹʔɔljɪɹ",
+                "per": 0.12121212121212122,
+                "pwed": 0.037990196078431376
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SA2.WAV",
+                "ground_truth": "oʊnæsmitikɛɹiinɔɪliɹæglaɪkðæt",
+                "prediction": "oʊnæskmitikæɹinɔɪliɹæɡlaɪkðæt",
+                "per": 0.14285714285714285,
+                "pwed": 0.10632183908045977
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI1573.WAV",
+                "ground_truth": "hɪzkæpinwəsθɪnænhægɹdinɪzbjuɾuflbutswɹwɔɹninʃæbi",
+                "prediction": "hɪzkæpinwəsθɪnhæɡɹdinizbjuɾiflbutswɹwɔɹninʃæbi",
+                "per": 0.10638297872340426,
+                "pwed": 0.0425531914893617
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI2203.WAV",
+                "ground_truth": "ðiɹizənzfɹðɪsdaɪvsimdfuliʃnaʊ",
+                "prediction": "ðəɹiznzfɹðistaɪvsimdfuliʃnaʊ",
+                "per": 0.13793103448275862,
+                "pwed": 0.04166666666666667
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI943.WAV",
+                "ground_truth": "ɹdʌkʃinmeɪfɔlfɑɹbəloʊəkspikeɪʃnts",
+                "prediction": "pɹdʌkʃnmeɪfɔlfɑɹbloʊɛkspɛkeɪʃəns",
+                "per": 0.21212121212121213,
+                "pwed": 0.10858585858585859
+            }
+        ],
+        "timestamp": "2024-12-05T08:25:24.980111"
+    },
+    {
+        "task_id": "dbf4642a-fb13-402c-8a74-cc41fc4be599",
+        "model": "speech31/wav2vec2-large-TIMIT-IPA",
+        "subset": "test",
+        "num_files": 1680,
+        "average_per": 0.4415425496841929,
+        "average_pwed": 0.18625930002594002,
+        "detailed_results": [
+            {
+                "file": "data/TEST/DR1/FAKS0/SA1.WAV",
+                "ground_truth": "ʃihædjɹdɑɹksuɾɪŋgɹisiwɑʃwɑɾɹʔɔljiɹ",
+                "prediction": "ʃihædjʊrdɑrksutɪngrisiwɑʃwɔtərɔljɪrrrɪrɪrʃ",
+                "per": 0.5757575757575758,
+                "pwed": 0.25
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SA2.WAV",
+                "ground_truth": "oʊnæsmitikɛɹiinɔɪliɹæglaɪkðæt",
+                "prediction": "doʊntæskmitɪkɛriənɔɪliræglaɪkðəttm",
+                "per": 0.35714285714285715,
+                "pwed": 0.172979797979798
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI1573.WAV",
+                "ground_truth": "hɪzkæpinwəsθɪnænhægɹdinɪzbjuɾuflbutswɹwɔɹninʃæbi",
+                "prediction": "hɪzkæptɪnwɑzθɪnəndhægərdəndhɪzbjutəfəlbutswərwɔrnəndʃæbi",
+                "per": 0.40425531914893614,
+                "pwed": 0.17500000000000004
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI2203.WAV",
+                "ground_truth": "ðiɹizənzfɹðɪsdaɪvsimdfuliʃnaʊ",
+                "prediction": "ðərizɪənzfərðɪstaɪvsimdfulɪʃnaʊaʊaʊ",
+                "per": 0.3793103448275862,
+                "pwed": 0.18928571428571428
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI943.WAV",
+                "ground_truth": "ɹdʌkʃinmeɪfɔlfɑɹbəloʊəkspikeɪʃnts",
+                "prediction": "prədəkʃənmeɪfɔlfɑrbɪloʊɛkspɛkteɪʃənzd",
+                "per": 0.3939393939393939,
+                "pwed": 0.13626126126126126
+            }
+        ],
+        "timestamp": "2024-12-05T09:36:14.568321"
+    },
+    {
+        "task_id": "912449a4-d7ed-4af4-b5be-5c2c57ec09ff",
+        "model": "Jubliano/wav2vec2-large-xls-r-300m-ipa-INTERNATIONAL1.5",
+        "subset": "test",
+        "num_files": 1680,
+        "average_per": 0.6318471187460027,
+        "average_pwed": 0.222932144739126,
+        "detailed_results": [
+            {
+                "file": "data/TEST/DR1/FAKS0/SA1.WAV",
+                "ground_truth": "ʃihædjɹdɑɹksuɾɪŋgɹisiwɑʃwɑɾɹʔɔljiɹ",
+                "prediction": "ʒihɛldjydɑrksydənrisiwɑswadərɑlhir",
+                "per": 0.5454545454545454,
+                "pwed": 0.11764705882352941
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SA2.WAV",
+                "ground_truth": "oʊnæsmitikɛɹiinɔɪliɹæglaɪkðæt",
+                "prediction": "dɑnraːstɪkmədəkaːrənoːjliralɪkaːn",
+                "per": 0.7857142857142857,
+                "pwed": 0.2341954022988506
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI1573.WAV",
+                "ground_truth": "hɪzkæpinwəsθɪnænhægɹdinɪzbjuɾuflbutswɹwɔɹninʃæbi",
+                "prediction": "xisʃktəʋɑstɪnɛnhɛɪɡərdɛnenzbjudəvɔlbutvɔːrʋɔrnənʃaːbi",
+                "per": 0.6595744680851063,
+                "pwed": 0.18382352941176472
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI2203.WAV",
+                "ground_truth": "ðiɹizənzfɹðɪsdaɪvsimdfuliʃnaʊ",
+                "prediction": "dərizənsvərdəstajfzimtvuləsna",
+                "per": 0.6206896551724138,
+                "pwed": 0.11781609195402297
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI943.WAV",
+                "ground_truth": "ɹdʌkʃinmeɪfɔlfɑɹbəloʊəkspikeɪʃnts",
+                "prediction": "pːdkəmeːvɑlvɑrbəloɛkspɛkteːʃəns",
+                "per": 0.5454545454545454,
+                "pwed": 0.2171717171717172
+            }
+        ],
+        "timestamp": "2024-12-05T10:17:21.331572"
+    },
+    {
+        "task_id": "c79df17e-2bb2-4253-ae26-f7cc6ab21265",
+        "model": "facebook/wav2vec2-xlsr-53-espeak-cv-ft",
+        "subset": "test",
+        "num_files": 1680,
+        "average_per": 0.348845592557092,
+        "average_pwed": 0.1386742019529415,
+        "detailed_results": [
+            {
+                "file": "data/TEST/DR1/FAKS0/SA1.WAV",
+                "ground_truth": "ʃihædjɹdɑɹksuɾɪŋgɹisiwɑʃwɑɾɹʔɔljiɹ",
+                "prediction": "ʃiːhædjɚdksuːtɪnɡɹiːsiwɑːʃwɑːɾɚɑːljɪ",
+                "per": 0.48484848484848486,
+                "pwed": 0.21338383838383837
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SA2.WAV",
+                "ground_truth": "oʊnæsmitikɛɹiinɔɪliɹæglaɪkðæt",
+                "prediction": "doːntæskmitəkæɹiənoɪliɹæɡlaɪkðæt",
+                "per": 0.32142857142857145,
+                "pwed": 0.12634408602150538
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI1573.WAV",
+                "ground_truth": "hɪzkæpinwəsθɪnænhægɹdinɪzbjuɾuflbutswɹwɔɹninʃæbi",
+                "prediction": "hɪzkæptənwʌzθɪnænhæɡɚdændhɪzbjuːɾɪfʊbuːtswɚwoːnəndʃæbi",
+                "per": 0.3617021276595745,
+                "pwed": 0.13095238095238093
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI2203.WAV",
+                "ground_truth": "ðiɹizənzfɹðɪsdaɪvsimdfuliʃnaʊ",
+                "prediction": "ðəɹiːzənzfɚðəsdɑːvsiːmdfuːlɪʃnæ",
+                "per": 0.3793103448275862,
+                "pwed": 0.12068965517241376
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI943.WAV",
+                "ground_truth": "ɹdʌkʃinmeɪfɔlfɑɹbəloʊəkspikeɪʃnts",
+                "prediction": "pɹədʌkʃənmeɪfɑːlfɑːbəloʊɛkspɛkteɪʃənz",
+                "per": 0.36363636363636365,
+                "pwed": 0.14404761904761906
+            }
+        ],
+        "timestamp": "2024-12-05T10:34:26.154521"
+    },
+    {
+        "task_id": "f36060e6-a746-44dc-a527-54995b270053",
+        "model": "ginic/hyperparam_tuning_1_wav2vec2-large-xlsr-buckeye-ipa",
+        "subset": "test",
+        "num_files": 1680,
+        "average_per": 0.2766466385175833,
+        "average_pwed": 0.10410683992600853,
+        "detailed_results": [
+            {
+                "file": "data/TEST/DR1/FAKS0/SA1.WAV",
+                "ground_truth": "ʃihædjɹdɑɹksuɾɪŋgɹisiwɑʃwɑɾɹʔɔljiɹ",
+                "prediction": "ʃihædjɹ̩dɑɹksuɾɪnɡɹeɪsiwɑʃwɔɾɹ̩ɔljiɹ",
+                "per": 0.24242424242424243,
+                "pwed": 0.09926470588235292
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SA2.WAV",
+                "ground_truth": "oʊnæsmitikɛɹiinɔɪliɹæglaɪkðæt",
+                "prediction": "doʊndæskmidɪkæɹiɛnɔɪliɹæɡlaɪkðæʔ",
+                "per": 0.32142857142857145,
+                "pwed": 0.14192708333333334
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI1573.WAV",
+                "ground_truth": "hɪzkæpinwəsθɪnænhægɹdinɪzbjuɾuflbutswɹwɔɹninʃæbi",
+                "prediction": "hɪzkæptɪnwʌzθɪnɛnhæɡɹ̩dɛnɪzbjuɾʌfl̩butswɹ̩wɔɹnɛnʃæbi",
+                "per": 0.2553191489361702,
+                "pwed": 0.05357142857142857
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI2203.WAV",
+                "ground_truth": "ðiɹizənzfɹðɪsdaɪvsimdfuliʃnaʊ",
+                "prediction": "ðʌɹizʌnzfɹ̩ðʌstaɪvsimdfulɪʃnaʊ",
+                "per": 0.20689655172413793,
+                "pwed": 0.01293103448275862
+            },
+            {
+                "file": "data/TEST/DR1/FAKS0/SI943.WAV",
+                "ground_truth": "ɹdʌkʃinmeɪfɔlfɑɹbəloʊəkspikeɪʃnts",
+                "prediction": "pɹʌdʌkʃʌnmeɪfɔlfɑɹbʌloʊɛkspɛkteɪʃʌns",
+                "per": 0.2727272727272727,
+                "pwed": 0.10416666666666667
+            }
+        ],
+        "timestamp": "2024-12-05T11:06:07.981224"
+    }
+]

fake_queue/tasks.json

@@ -0,0 +1,74 @@
+[
+    {
+        "id": "721b4c64-a825-42d3-bb0a-bdff9ee1ed0f",
+        "model": "facebook/wav2vec2-lv-60-espeak-cv-ft",
+        "subset": "test",
+        "submission_name": "facebook espeak",
+        "github_url": "https://github.com/facebookresearch/fairseq/blob/main/examples/wav2vec/README.md",
+        "status": "completed",
+        "submitted_at": "2024-12-05T07:19:03.076292"
+    },
+    {
+        "id": "d6fe0956-b5b4-4105-835e-8dee1872ee4d",
+        "model": "KoelLabs/xlsr-timit-b0",
+        "subset": "test",
+        "submission_name": "english phoneme model",
+        "github_url": "https://github.com/KoelLabs/",
+        "status": "completed",
+        "submitted_at": "2024-12-05T08:12:40.161444"
+    },
+    {
+        "id": "dbf4642a-fb13-402c-8a74-cc41fc4be599",
+        "model": "speech31/wav2vec2-large-TIMIT-IPA",
+        "subset": "test",
+        "submission_name": "speech 31 model",
+        "github_url": "https://huggingface.co/speech31/wav2vec2-large-TIMIT-IPA2",
+        "status": "completed",
+        "submitted_at": "2024-12-05T09:13:45.315361"
+    },
+    {
+        "id": "4e3b80be-b255-47f2-b4ae-18a12e232e8a",
+        "model": "Jubliano/wav2vec2-large-xls-r-300m-ipa-INTERNATIONAL1.5",
+        "subset": "test",
+        "submission_name": "Jubliano model",
+        "github_url": "https://huggingface.co/Jubliano/wav2vec2-large-xls-r-300m-ipa-INTERNATIONAL1.5WithoutSpaces/tree/d5312009d8e620b183c334dfdd9ffc6b4f06f8c1",
+        "status": "processing",
+        "submitted_at": "2024-12-05T09:36:14.571930"
+    },
+    {
+        "id": "912449a4-d7ed-4af4-b5be-5c2c57ec09ff",
+        "model": "Jubliano/wav2vec2-large-xls-r-300m-ipa-INTERNATIONAL1.5",
+        "subset": "test",
+        "submission_name": "jubiliano model wav2vec2",
+        "github_url": "https://huggingface.co/Jubliano/wav2vec2-large-xls-r-300m-ipa-INTERNATIONAL1.5WithoutSpaces/tree/d5312009d8e620b183c334dfdd9ffc6b4f06f8c1",
+        "status": "completed",
+        "submitted_at": "2024-12-05T10:01:40.502935"
+    },
+    {
+        "id": "c79df17e-2bb2-4253-ae26-f7cc6ab21265",
+        "model": "facebook/wav2vec2-xlsr-53-espeak-cv-ft",
+        "subset": "test",
+        "submission_name": "xlsr 53 model",
+        "github_url": "https://github.com/facebookresearch/fairseq/blob/main/examples/wav2vec/README.md",
+        "status": "completed",
+        "submitted_at": "2024-12-05T10:18:37.408664"
+    },
+    {
+        "id": "f36060e6-a746-44dc-a527-54995b270053",
+        "model": "ginic/hyperparam_tuning_1_wav2vec2-large-xlsr-buckeye-ipa",
+        "subset": "test",
+        "submission_name": "ginic model wav2vec2 finetuned on buckeye",
+        "github_url": "https://huggingface.co/ginic/vary_individuals_old_only_1_wav2vec2-large-xlsr-buckeye-ipa",
+        "status": "completed",
+        "submitted_at": "2024-12-05T10:36:02.340422"
+    },
+    {
+        "id": "abf6c247-9faf-46ef-b0fa-25f2669da922",
+        "model": "KoelLabs/xlsr-timit-a0",
+        "subset": "test",
+        "submission_name": "Koel Labs early version of finetuned model ",
+        "github_url": "https://github.com/KoelLabs/ML",
+        "status": "processing",
+        "submitted_at": "2024-12-05T11:08:23.663553"
+    }
+]