Upload 7 files

.gitattributes

@@ -1,35 +1,37 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+conformer.png filter=lfs diff=lfs merge=lfs -text
+whisper.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,13 +1,13 @@
----
-title: Multilingual Indonesia Whisper Model
-emoji: 🏃
-colorFrom: gray
-colorTo: indigo
-sdk: gradio
-sdk_version: 5.29.1
-app_file: app.py
-pinned: false
-short_description: a fine tuned javaneses and sundanese Whisper model.
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Multilingual Indonesia Whisper Model
+emoji: 🏃
+colorFrom: gray
+colorTo: indigo
+sdk: gradio
+sdk_version: 5.29.1
+app_file: app.py
+pinned: false
+short_description: a fine tuned javaneses and sundanese Whisper model.
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,180 @@
+import torch  # type: ignore
+import numpy as np  # type: ignore
+import gradio as gr  # type: ignore
+from transformers import pipeline
+
+# Load fine-tuned Whisper model
+transcriber = pipeline("automatic-speech-recognition", model="OwLim/whisper-java-SLR41-SLR35")
+
+def transcribe(audio):
+    sr, waveform = audio
+    # Change into Mono Audio
+    if waveform.ndim > 1:
+        waveform = waveform.mean(axis=1)
+
+    # Normalisasi
+    waveform = waveform.astype(np.float32)
+    waveform /= np.max(np.abs(waveform))
+
+    return transcriber({
+        "sampling_rate" : sr,
+        "raw" : waveform
+    })["text"]
+
+def clear():
+    return None, ""
+
+# --- Tab 1: Transcribe ---
+with gr.Blocks() as tab_transcribe:
+    with gr.Row():
+        with gr.Column(scale=1):
+            audio_input = gr.Audio(sources="microphone", label="Record Your Voice")
+            with gr.Row():
+                subBtn = gr.Button("Submit", variant="primary")
+                clrBtn = gr.ClearButton(variant="stop")
+
+        with gr.Column(scale=1):
+            output_text = gr.Textbox(label="Transcription", placeholder="Waiting for Input", lines=3)
+
+    subBtn.click(fn=transcribe, inputs=audio_input, outputs=output_text)
+    clrBtn.click(fn=clear, outputs=[audio_input, output_text])
+
+# --- Tab 2: Penjelasan Model Fine-Tuned ---
+with gr.Blocks() as tab_background:
+    gr.HTML("""
+    <h3>Latar Belakang Project:</h3>
+    <p>
+        Pada project kita kali ini, kami ingin membuat suatu model AI Speech Recognition yang mampu untuk menerima, mengenali dan memproses input berupa ucapan lisan multilingual termasuk dengan adanya bahasa lokal (seperti bahasa daerah). Projek kami didasarkan dengan kurangnya pengaplikasian bidang Speech Recognition pada low-resource language atau bahasa-bahasa yang memiliki data atau sumber daya yang relatif lebih sedikit. Kami ingin membuat model yang dapat mendeteksi pengguna baik berbahasa inggris, berbahasa indonesia maupun berbahasa daerah seperti bahasa jawa.
+    </p>
+    <br>    
+    <p>
+        Dengan adanya multi-lingual speech recognition, pengolahan lisan dalam bahasa daerah seperti di Indonesia akan lebih terbantu dan semakin banyak pula. Terlebih lagi, pengolahan lisan dalam bahasa daerah masih sedikit dan kurang diperhatikan walaupun di Indonesia sendiri memiliki lebih dari 500 ragam banyaknya. Dari hal ini, kami ingin mengembangkan dua model AI dalam ranah Speech Recognition, yaitu Conformer dan Whisper untuk dapat belajar dan memproses bahasa multilingual.
+
+        Model yang telah kami fine tune merupakan hasil <b>fine-tuning dari Whisper dan Conformer</b> untuk mendukung bahasa lokal di Indonesia, khususnya bahasa Jawa dan Sunda.
+        Model dilatih menggunakan kombinasi dataset <b>OpenSLR</b> berikut:
+            <br>
+            <a href="https://openslr.org/35/" target="_blank" style="text-decoration:none;> 
+                    <b>SLR35</b> - Large Javanese ASR
+            </a>
+
+            <br>
+            <a href="https://openslr.org/41/" target="_blank" style="text-decoration:none;">             
+                    <b>SLR41</b> - High quality TTS data for Javanese
+            </a>        
+
+            <br>
+            <a href="https://openslr.org/36" target="_blank" style="text-decoration:none;">             
+                    <b>SLR36</b>    
+                    <b>SLR44</b> - Bilingual speech datasets
+            </a>
+        
+                <br>
+        Model ini diharapkan bisa meningkatkan akurasi untuk bahasa yang sebelumnya kurang terwakili dalam model global.
+    </p>
+    <h3>Tujuan Project:</h3>
+    <ul>
+        <li>Dapat memahami ucapan lisan multilingual termasuk dengan bahasa <i>low-resource language</i>.</li>
+        <li>Ikut serta dalam pengembangan teknologi dalam pelatihan model pada <i>low-resource language</i>.</li>
+        <li>Berpartisipasi dalam pelestarian dan pembudidayaan bahasa-bahasa daerah di Indonesia yang kurang mendapatkan perhatian.</li>
+    </ul>
+    """)
+
+# --- Tab 3: Arsitektur Model ---
+with gr.Blocks() as tab_architecture:
+    gr.Markdown("### 🧠 Whisper Architecture")
+    with gr.Row():
+        with gr.Column():
+            gr.HTML("""
+            <div>
+                <p>
+                    Whisper adalah model Automatic Speech Recognition (ASR) open-source yang dikembangkan oleh OpenAI. 
+                    Model ini dilatih menggunakan <strong>680,000 jam</strong> data audio multilingual dan multitask, 
+                    termasuk data yang memiliki noise dan hasil transkripsi otomatis untuk meningkatkan robustness.
+                </p>
+                <p>
+                    Whisper mampu mentranskrip audio dengan <em>background noise</em>, serta memahami berbagai aksen dan 
+                    bahasa secara efektif.
+                </p>
+            </div>
+            """)
+        with gr.Column():
+            gr.Image("whisper.png", show_label=False, show_download_button=False)
+
+    gr.Markdown("### 🔊 Conformer Architecture")
+    with gr.Row():
+        with gr.Column():
+            gr.HTML("""
+            <div>
+                <p>
+                    <strong>Conformer (Convolutional Transformer)</strong> adalah arsitektur deep learning yang dirancang khusus untuk pengolahan sinyal suara, seperti speech recognition.
+                </p>
+                <br>
+                <p>Model ini menggabungkan dua komponen utama:</p>
+                <ul>
+                    <li><strong>Transformer:</strong> Menangkap hubungan global dalam data, seperti relasi antar kata dalam kalimat.</li>
+                    <li><strong>CNN (Convolutional Neural Network):</strong> Menangkap pola lokal seperti fonem atau suku kata dalam suara.</li>
+                </ul>
+                <p>
+                    Dengan kombinasi ini, Conformer dapat memahami baik konteks global maupun detail lokal dari sinyal suara secara lebih efektif.
+                </p>
+            </div>
+            """)
+        with gr.Column():
+            gr.Image("conformer.png", show_label=False, show_download_button=False)
+
+
+# --- Tab 4: Tabel Hasil Evaluasi ---
+with gr.Blocks() as tab_results:
+    gr.HTML("""
+    <h1 style='text-align:center;'>📊 Hasil Evaluasi Model</h1>
+    <table style="width:80%; margin:20px auto; border-collapse:separate; border-spacing:0; font-family:sans-serif;">
+        <thead style="background-color:#eb7434; color:white;">
+            <tr>
+                <th style="padding:15px 20px; text-align:center;">Dataset</th>
+                <th style="padding:15px 20px; text-align:center;">WER (Word Error Rate)</th>
+                <th style="padding:15px 20px; text-align:center;">CER (Character Error Rate)</th>
+            </tr>
+        </thead>
+        <tbody>
+            <tr>
+                <td style="padding:15px 20px; text-align:center;">SLR35 (Javanese)</td>
+                <td style="padding:15px 20px; text-align:center;">15.2%</td>
+                <td style="padding:15px 20px; text-align:center;">9.8%</td>
+            </tr>
+            <tr>
+                <td style="padding:15px 20px; text-align:center;">SLR41 (Indonesian)</td>
+                <td style="padding:15px 20px; text-align:center;">12.4%</td>
+                <td style="padding:15px 20px; text-align:center;">8.1%</td>
+            </tr>
+            <tr>
+                <td style="padding:15px 20px; text-align:center;">SLR44 (Multilingual)</td>
+                <td style="padding:15px 20px; text-align:center;">17.3%</td>
+                <td style="padding:15px 20px; text-align:center;">11.0%</td>
+            </tr>
+        </tbody>
+    </table>
+    """)
+
+# --- Tab 5: Fine-tuning Info ---
+with gr.Blocks() as tab_authors:
+    gr.HTML("""
+    <h1 style='text-align:center;'>👨‍💻 Fine-Tuning Information</h1>
+    <p style='text-align:center;'>
+        Model ini di-fine-tune oleh <b>OwLim</b> menggunakan framework Hugging Face Transformers dan PyTorch Lightning.<br><br>
+        Semua pelatihan dilakukan di GPU NVIDIA RTX 3090 dengan total 40+ jam data suara.
+    </p>
+    <p style='text-align:center;'>
+        Lihat model di <a href='https://huggingface.co/OwLim/whisper-java-SLR41-SLR35' target='_blank'>Hugging Face Model Hub</a>.
+    </p>
+    """)
+
+# Gabungkan semua tabs ke dalam aplikasi utama
+demo = gr.TabbedInterface(
+    [tab_transcribe, tab_background, tab_architecture, tab_results, tab_authors],
+    ["Transcribe", "Latar Belakang", "Arsitektur", "Evaluasi", "Fine-Tuned By"],
+    theme=gr.themes.Soft(),
+    title="Whisper VS Conformer Model   "
+)
+
+if __name__ == "__main__":
+    demo.launch()

requirement.txt

@@ -0,0 +1,4 @@
+torch
+numpy
+torchaudio
+transformers