import torch
from transformers import pipeline
import numpy as np
import gradio as gr
def _grab_best_device(use_gpu=True):
if torch.cuda.device_count() > 0 and use_gpu:
device = "cuda"
else:
device = "cpu"
return device
device = _grab_best_device()
default_model_per_language = {
"spanish": "facebook/mms-tts-spa",
"tamil": "facebook/mms-tts-tam",
"gujarati": "facebook/mms-tts-guj",
"marathi": "facebook/mms-tts-mar",
"english": "kakao-enterprise/vits-ljs",
}
models_per_language = {
"english": [
"ylacombe/vits_ljs_midlands_male_monospeaker",
],
"spanish": [
"ylacombe/mms-spa-finetuned-chilean-monospeaker",
],
"tamil": [
"ylacombe/mms-tam-finetuned-monospeaker",
],
"gujarati" : ["ylacombe/mms-guj-finetuned-monospeaker"],
"marathi": ["ylacombe/mms-mar-finetuned-monospeaker"]
}
HUB_PATH = "ylacombe/vits_ljs_midlands_male_monospeaker"
pipe_dict = {
"current_model": "ylacombe/vits_ljs_midlands_male_monospeaker",
"pipe": pipeline("text-to-speech", model=HUB_PATH, device=0),
"original_pipe": pipeline("text-to-speech", model=default_model_per_language["english"], device=0),
"language": "english",
}
title = """
# Explore MMS finetuning
## Or how to access truely multilingual TTS
Massively Multilingual Speech (MMS) models are light-weight, low-latency TTS models based on the [VITS architecture](https://huggingface.co/docs/transformers/model_doc/vits).
Meta's [MMS](https://arxiv.org/abs/2305.13516) project, aiming to provide speech technology across a diverse range of languages. You can find more details about the supported languages and their ISO 639-3 codes in the [MMS Language Coverage Overview](https://dl.fbaipublicfiles.com/mms/misc/language_coverage_mms.html),
and see all MMS-TTS checkpoints on the Hugging Face Hub: [facebook/mms-tts](https://huggingface.co/models?sort=trending&search=facebook%2Fmms-tts).
Coupled with the right data and the right training recipe, you can get an excellent finetuned version of every MMS checkpoints in **20 minutes** with as little as **80 to 150 samples**.
Training recipe available in this [github repository](https://github.com/ylacombe/finetune-hf-vits)!
"""
max_speakers = 15
# Inference
def generate_audio(text, model_id, language):
if pipe_dict["language"] != language:
gr.Warning(f"Language has changed - loading new default model: {default_model_per_language[language]}")
pipe_dict["language"] = language
pipe_dict["original_pipe"] = pipeline("text-to-speech", model=default_model_per_language[language], device=0)
if pipe_dict["current_model"] != model_id:
gr.Warning("Model has changed - loading new model")
pipe_dict["pipe"] = pipeline("text-to-speech", model=model_id, device=0)
pipe_dict["current_model"] = model_id
num_speakers = pipe_dict["pipe"].model.config.num_speakers
out = []
# first generate original model result
output = pipe_dict["original_pipe"](text)
output = gr.Audio(value = (output["sampling_rate"], output["audio"].squeeze()), type="numpy", autoplay=False, label=f"Non finetuned model prediction {default_model_per_language[language]}", show_label=True,
visible=True)
out.append(output)
if num_speakers>1:
for i in range(min(num_speakers, max_speakers - 1)):
forward_params = {"speaker_id": i}
output = pipe_dict["pipe"](text, forward_params=forward_params)
output = gr.Audio(value = (output["sampling_rate"], output["audio"].squeeze()), type="numpy", autoplay=False, label=f"Generated Audio - speaker {i}", show_label=True,
visible=True)
out.append(output)
out.extend([gr.Audio(visible=False)]*(max_speakers-num_speakers))
else:
output = pipe_dict["pipe"](text)
output = gr.Audio(value = (output["sampling_rate"], output["audio"].squeeze()), type="numpy", autoplay=False, label="Generated Audio - Mono speaker", show_label=True,
visible=True)
out.append(output)
out.extend([gr.Audio(visible=False)]*(max_speakers-2))
return out
css = """
#container{
margin: 0 auto;
max-width: 80rem;
}
#intro{
max-width: 100%;
text-align: center;
margin: 0 auto;
}
"""
# Gradio blocks demo
with gr.Blocks(css=css) as demo_blocks:
gr.Markdown(title, elem_id="intro")
with gr.Row():
with gr.Column():
inp_text = gr.Textbox(label="Input Text", info="What sentence would you like to synthesise?")
btn = gr.Button("Generate Audio!")
language = gr.Dropdown(
default_model_per_language.keys(),
value = "spanish",
label = "language",
info = "Language that you want to test"
)
model_id = gr.Dropdown(
models_per_language["spanish"],
value="ylacombe/mms-spa-finetuned-chilean-monospeaker",
label="Model",
info="Model you want to test",
)
with gr.Column():
outputs = []
for i in range(max_speakers):
out_audio = gr.Audio(type="numpy", autoplay=False, label=f"Generated Audio - speaker {i}", show_label=True, visible=False)
outputs.append(out_audio)
with gr.Accordion("Datasets and models details", open=False):
gr.Markdown("""
For each language, we used 100 to 150 samples of a single speaker to finetune the model.
### Spanish
* **Model**: [Spanish MMS TTS](https://huggingface.co/facebook/mms-tts-spa).
* **Datasets**:
- [Chilean Spanish TTS dataset](https://huggingface.co/datasets/ylacombe/google-chilean-spanish).
### Tamil
* **Model**: [Tamil MMS TTS](https://huggingface.co/facebook/mms-tts-tam).
* **Datasets**:
- [Tamil TTS dataset](https://huggingface.co/datasets/ylacombe/google-tamil).
### Gujarati
* **Model**: [Gujarati MMS TTS](https://huggingface.co/facebook/mms-tts-guj).
* **Datasets**:
- [Gujarati TTS dataset](https://huggingface.co/datasets/ylacombe/google-gujarati).
### Marathi
* **Model**: [Marathi MMS TTS](https://huggingface.co/facebook/mms-tts-mar).
* **Datasets**:
- [Marathi TTS dataset](https://huggingface.co/datasets/ylacombe/google-chilean-marathi).
### English
* **Model**: [VITS-ljs](https://huggingface.co/kakao-enterprise/vits-ljs)
* **Dataset**: [British Isles Accent](https://huggingface.co/datasets/ylacombe/english_dialects). For each accent, we used 100 to 150 samples of a single speaker to finetune [VITS-ljs](https://huggingface.co/kakao-enterprise/vits-ljs).
""")
with gr.Accordion("Run VITS and MMS with transformers", open=False):
gr.Markdown(
"""
```bash
pip install transformers
```
```py
from transformers import pipeline
import scipy
pipe = pipeline("text-to-speech", model="kakao-enterprise/vits-ljs", device=0)
results = pipe("A cinematic shot of a baby racoon wearing an intricate italian priest robe")
# write to a wav file
scipy.io.wavfile.write("audio_vits.wav", rate=results["sampling_rate"], data=results["audio"].squeeze())
```
"""
)
language.change(lambda language: gr.Dropdown(
models_per_language[language],
value=models_per_language[language][0],
label="Model",
info="Model you want to test",
),
language,
model_id
)
btn.click(generate_audio, [inp_text, model_id, language], outputs)
demo_blocks.queue().launch()