Create app.py

app.py

@@ -0,0 +1,271 @@
+import gc
+import json
+import tempfile
+
+import gradio as gr
+import numpy as np
+import soundfile as sf
+import torch
+import torchaudio
+from cached_path import cached_path
+from ruaccent import RUAccent
+import onnx_asr
+
+from f5_tts.infer.utils_infer import (
+    infer_process,
+    load_model,
+    load_vocoder,
+    preprocess_ref_audio_text,
+    remove_silence_for_generated_wav,
+    save_spectrogram,
+    tempfile_kwargs,
+)
+from f5_tts.model import DiT
+
+
+# --- Model configuration ---
+MODEL_CFG = dict(dim=1024, depth=22, heads=16, ff_mult=2, text_dim=512, conv_layers=4)
+
+# Paths for all models (впишите свои пути)
+MODEL_PATHS = {
+    "ESpeech-TTS-1 [RL] V2": "stripped_states/espeech_tts_rlv2.pt",
+    "ESpeech-TTS-1 [RL] V1": "stripped_states/espeech_tts_rlv1.pt",
+    "ESpeech-TTS-1 [SFT] 95K": "stripped_states/espeech_tts_95k.pt",
+    "ESpeech-TTS-1 [SFT] 265K": "stripped_states/espeech_tts_256k.pt",
+    "ESpeech-TTS-1 PODCASTER [SFT]": "stripped_states/espeech_tts_podcaster.pt"
+}
+
+# Shared vocabulary path (одинаковый для всех моделей)
+VOCAB_PATH = "/media/denis/work/f5tts/F5-TTS/base_checkpoint1/vocab.txt"
+
+# Load vocoder (shared)
+vocoder = load_vocoder()
+
+# Dictionary to store loaded models
+loaded_models = {}
+
+# Initialize RUAccent
+print("Loading RUAccent...")
+accentizer = RUAccent()
+accentizer.load(omograph_model_size='turbo3.1', use_dictionary=True, tiny_mode=False)
+print("RUAccent loaded successfully.")
+
+# Initialize ASR model
+print("Loading ASR model...")
+asr_model = onnx_asr.load_model("nemo-fastconformer-ru-rnnt")
+print("ASR model loaded successfully.")
+
+# Load all models at startup
+print("Loading models...")
+for model_name, model_path in MODEL_PATHS.items():
+    print(f"Loading {model_name}...")
+    loaded_models[model_name] = load_model(
+        DiT, 
+        MODEL_CFG, 
+        model_path, 
+        vocab_file=VOCAB_PATH
+    )
+    print(f"{model_name} loaded successfully.")
+
+print("All models loaded successfully.")
+
+
+def synthesize(
+    model_choice,
+    ref_audio,
+    ref_text,
+    gen_text,
+    remove_silence,
+    seed,
+    cross_fade_duration=0.15,
+    nfe_step=32,
+    speed=1.0,
+):
+    if not ref_audio:
+        gr.Warning("Please provide reference audio.")
+        return None, None, ref_text
+
+    if seed < 0 or seed > 2**31 - 1:
+        seed = np.random.randint(0, 2**31 - 1)
+    torch.manual_seed(seed)
+
+    if not gen_text.strip():
+        gr.Warning("Please enter text to generate.")
+        return None, None, ref_text
+
+    # If reference text is empty, use ASR to transcribe reference audio
+    # If reference text is empty, use ASR to transcribe reference audio
+    if not ref_text.strip():
+        gr.Info("Reference text is empty. Running ASR to transcribe reference audio...")
+        try:
+            # Load audio data from Gradio (correct order: waveform first, then sample_rate)
+            waveform, sample_rate = torchaudio.load(ref_audio)
+            
+            # Convert tensor to numpy
+            waveform = waveform.numpy()
+            
+            # Convert to the format expected by onnx-asr
+            if waveform.dtype == np.int16:
+                waveform = waveform / 2**15
+            elif waveform.dtype == np.int32:
+                waveform = waveform / 2**31
+            elif waveform.dtype == np.float32 or waveform.dtype == np.float64:
+                pass  # already in the right range
+            
+            # Convert to mono if stereo
+            if waveform.ndim == 2:
+                waveform = waveform.mean(axis=0)  # average across channels (first dimension)
+            elif waveform.ndim == 1:
+                pass  # already mono
+            else:
+                waveform = waveform.squeeze()
+            
+            # Run ASR on the audio data directly
+            transcribed_text = asr_model.recognize(waveform, sample_rate=sample_rate)
+            ref_text = transcribed_text
+            gr.Info(f"ASR transcription: {ref_text}")
+            
+        except Exception as e:
+            gr.Warning(f"ASR transcription failed: {str(e)}")
+            return None, None, ref_text
+
+    # Apply accent marks to reference text and generation text
+    processed_ref_text = accentizer.process_all(ref_text) if ref_text.strip() else ref_text
+    processed_gen_text = accentizer.process_all(gen_text)
+
+    # Select model based on choice
+    model = loaded_models[model_choice]
+    
+    # Preprocess reference audio and text
+    ref_audio, processed_ref_text = preprocess_ref_audio_text(
+        ref_audio, 
+        processed_ref_text, 
+        show_info=gr.Info
+    )
+
+    # Generate speech
+    final_wave, final_sample_rate, combined_spectrogram = infer_process(
+        ref_audio,
+        processed_ref_text,
+        processed_gen_text,
+        model,
+        vocoder,
+        cross_fade_duration=cross_fade_duration,
+        nfe_step=nfe_step,
+        speed=speed,
+        show_info=gr.Info,
+        progress=gr.Progress(),
+    )
+
+    # Remove silence if requested
+    if remove_silence:
+        with tempfile.NamedTemporaryFile(suffix=".wav", **tempfile_kwargs) as f:
+            temp_path = f.name
+            sf.write(temp_path, final_wave, final_sample_rate)
+            remove_silence_for_generated_wav(temp_path)
+            final_wave, _ = torchaudio.load(temp_path)
+            final_wave = final_wave.squeeze().cpu().numpy()
+
+    # Save spectrogram
+    with tempfile.NamedTemporaryFile(suffix=".png", **tempfile_kwargs) as tmp_spectrogram:
+        spectrogram_path = tmp_spectrogram.name
+    save_spectrogram(combined_spectrogram, spectrogram_path)
+
+    return (final_sample_rate, final_wave), spectrogram_path, processed_ref_text
+
+
+# --- Gradio interface ---
+with gr.Blocks(title="ESpeech-TTS") as app:
+    gr.Markdown("# ESpeech-TTS")
+    gr.Markdown("Text-to-Speech synthesis system with multiple model variants")
+    gr.Markdown("💡 **Tip:** If you leave the Reference Text empty, it will be automatically transcribed using ASR and then processed with accent marks!")
+
+    with gr.Row():
+        model_choice = gr.Dropdown(
+            choices=list(MODEL_PATHS.keys()),
+            label="Select Model",
+            value="ESpeech-TTS-1 [RL] V2",
+            interactive=True
+        )
+    
+    with gr.Row():
+        with gr.Column():
+            ref_audio_input = gr.Audio(
+                label="Reference Audio", 
+                type="filepath"
+            )
+            ref_text_input = gr.Textbox(
+                label="Reference Text",
+                lines=2,
+                placeholder="Enter the transcription of the reference audio... (leave empty for automatic ASR transcription)"
+            )
+        
+        with gr.Column():
+            gen_text_input = gr.Textbox(
+                label="Text to Generate",
+                lines=5,
+                max_lines=20,
+                placeholder="Enter the text you want to synthesize..."
+            )
+            
+    with gr.Row():
+        with gr.Column():
+            with gr.Accordion("Advanced Settings", open=False):
+                seed_input = gr.Number(
+                    label="Seed (-1 for random)",
+                    value=-1,
+                    precision=0
+                )
+                remove_silence = gr.Checkbox(
+                    label="Remove Silences",
+                    value=False
+                )
+                speed_slider = gr.Slider(
+                    label="Speed",
+                    minimum=0.3,
+                    maximum=2.0,
+                    value=1.0,
+                    step=0.1
+                )
+                nfe_slider = gr.Slider(
+                    label="NFE Steps (higher = better quality, slower)",
+                    minimum=4,
+                    maximum=64,
+                    value=48,
+                    step=2
+                )
+                cross_fade_slider = gr.Slider(
+                    label="Cross-Fade Duration (s)",
+                    minimum=0.0,
+                    maximum=1.0,
+                    value=0.15,
+                    step=0.01
+                )
+    
+    generate_btn = gr.Button("🎤 Generate Speech", variant="primary", size="lg")
+    
+    with gr.Row():
+        audio_output = gr.Audio(label="Generated Audio", type="numpy")
+        spectrogram_output = gr.Image(label="Spectrogram", type="filepath")
+    
+    
+    generate_btn.click(
+        synthesize,
+        inputs=[
+            model_choice,
+            ref_audio_input,
+            ref_text_input,
+            gen_text_input,
+            remove_silence,
+            seed_input,
+            cross_fade_slider,
+            nfe_slider,
+            speed_slider,
+        ],
+        outputs=[audio_output, spectrogram_output, ref_text_input]
+    )
+
+
+if __name__ == "__main__":
+    #app.launch(server_name="0.0.0.0", server_port=7860)
+    app.launch()