|
import spaces |
|
import gradio as gr |
|
import torch |
|
from transformers.models.speecht5.number_normalizer import EnglishNumberNormalizer |
|
from string import punctuation |
|
import re |
|
|
|
|
|
from parler_tts import ParlerTTSForConditionalGeneration |
|
from transformers import AutoTokenizer, AutoFeatureExtractor, set_seed |
|
|
|
device = "cuda:0" if torch.cuda.is_available() else "cpu" |
|
|
|
|
|
repo_id = "parler-tts/parler-tts-mini-v1" |
|
repo_id_large = "parler-tts/parler-tts-large-v1" |
|
|
|
model = ParlerTTSForConditionalGeneration.from_pretrained(repo_id).to(device) |
|
model_large = ParlerTTSForConditionalGeneration.from_pretrained(repo_id_large).to(device) |
|
tokenizer = AutoTokenizer.from_pretrained(repo_id) |
|
feature_extractor = AutoFeatureExtractor.from_pretrained(repo_id) |
|
|
|
|
|
SAMPLE_RATE = feature_extractor.sampling_rate |
|
SEED = 42 |
|
|
|
default_text = "All of the data, pre-processing, training code, and weights are released publicly under a permissive license, enabling the community to build on our work and develop their own powerful models." |
|
default_description = "Laura's voice is monotone yet slightly fast in delivery, with a very close recording that almost has no background noise." |
|
examples = [ |
|
[ |
|
"This version introduces speaker consistency across generations, characterized by their name. For example, Jon, Lea, Gary, Jenna, Mike and Laura.", |
|
"Gary's voice is monotone yet slightly fast in delivery, with a very close recording that has no background noise.", |
|
None, |
|
], |
|
[ |
|
'''There's 34 speakers. To take advantage of this, simply adapt your text description to specify which speaker to use: "Mike speaks animatedly...".''', |
|
"Gary speaks slightly animatedly and slightly slowly in delivery, with a very close recording that has no background noise.", |
|
None |
|
], |
|
[ |
|
"'This is the best time of my life, Bartley,' she said happily.", |
|
"A female speaker delivers a slightly expressive and animated speech with a moderate speed. The recording features a low-pitch voice and slight background noise, creating a close-sounding audio experience.", |
|
None, |
|
], |
|
[ |
|
"Montrose also, after having experienced still more variety of good and bad fortune, threw down his arms, and retired out of the kingdom.", |
|
"A man voice speaks slightly slowly with very noisy background, carrying a low-pitch tone and displaying a touch of expressiveness and animation. The sound is very distant, adding an air of intrigue.", |
|
None |
|
], |
|
[ |
|
"Once upon a time, in the depth of winter, when the flakes of snow fell like feathers from the clouds, a queen sat sewing at her pal-ace window, which had a carved frame of black wood.", |
|
"In a very poor recording quality, a female speaker delivers her slightly expressive and animated words with a fast pace. There's high level of background noise and a very distant-sounding reverberation. Her voice is slightly higher pitched than average.", |
|
None, |
|
], |
|
] |
|
|
|
number_normalizer = EnglishNumberNormalizer() |
|
|
|
def preprocess(text): |
|
text = number_normalizer(text).strip() |
|
text = text.replace("-", " ") |
|
if text[-1] not in punctuation: |
|
text = f"{text}." |
|
|
|
abbreviations_pattern = r'\b[A-Z][A-Z\.]+\b' |
|
|
|
def separate_abb(chunk): |
|
chunk = chunk.replace(".","") |
|
print(chunk) |
|
return " ".join(chunk) |
|
|
|
abbreviations = re.findall(abbreviations_pattern, text) |
|
for abv in abbreviations: |
|
if abv in text: |
|
text = text.replace(abv, separate_abb(abv)) |
|
return text |
|
|
|
@spaces.GPU |
|
def gen_tts(text, description, use_large=False): |
|
inputs = tokenizer(description.strip(), return_tensors="pt").to(device) |
|
prompt = tokenizer(preprocess(text), return_tensors="pt").to(device) |
|
|
|
set_seed(SEED) |
|
if use_large: |
|
generation = model_large.generate( |
|
input_ids=inputs.input_ids, prompt_input_ids=prompt.input_ids, attention_mask=inputs.attention_mask, prompt_attention_mask=prompt.attention_mask, do_sample=True, temperature=1.0 |
|
) |
|
else: |
|
generation = model.generate( |
|
input_ids=inputs.input_ids, prompt_input_ids=prompt.input_ids, attention_mask=inputs.attention_mask, prompt_attention_mask=prompt.attention_mask, do_sample=True, temperature=1.0 |
|
) |
|
audio_arr = generation.cpu().numpy().squeeze() |
|
|
|
return SAMPLE_RATE, audio_arr |
|
|
|
|
|
css = """ |
|
#share-btn-container { |
|
display: flex; |
|
padding-left: 0.5rem !important; |
|
padding-right: 0.5rem !important; |
|
background-color: #000000; |
|
justify-content: center; |
|
align-items: center; |
|
border-radius: 9999px !important; |
|
width: 13rem; |
|
margin-top: 10px; |
|
margin-left: auto; |
|
flex: unset !important; |
|
} |
|
#share-btn { |
|
all: initial; |
|
color: #ffffff; |
|
font-weight: 600; |
|
cursor: pointer; |
|
font-family: 'IBM Plex Sans', sans-serif; |
|
margin-left: 0.5rem !important; |
|
padding-top: 0.25rem !important; |
|
padding-bottom: 0.25rem !important; |
|
right:0; |
|
} |
|
#share-btn * { |
|
all: unset !important; |
|
} |
|
#share-btn-container div:nth-child(-n+2){ |
|
width: auto !important; |
|
min-height: 0px !important; |
|
} |
|
#share-btn-container .wrap { |
|
display: none !important; |
|
} |
|
""" |
|
with gr.Blocks(css=css) as block: |
|
gr.HTML( |
|
""" |
|
<div style="text-align: center; max-width: 700px; margin: 0 auto;"> |
|
<div |
|
style=" |
|
display: inline-flex; align-items: center; gap: 0.8rem; font-size: 1.75rem; |
|
" |
|
> |
|
<h1 style="font-weight: 900; margin-bottom: 7px; line-height: normal;"> |
|
Parler-TTS 🗣️ |
|
</h1> |
|
</div> |
|
</div> |
|
""" |
|
) |
|
gr.HTML( |
|
f""" |
|
<p><a href="https://github.com/huggingface/parler-tts"> Parler-TTS</a> is a training and inference library for |
|
high-fidelity text-to-speech (TTS) models.</p> |
|
<p>The models demonstrated here, Parler-TTS <a href="https://huggingface.co/parler-tts/parler-tts-mini-v1">Mini v1</a> and <a href="https://huggingface.co/parler-tts/parler-tts-large-v1">Large v1</a>, |
|
are trained using 45k hours of narrated English audiobooks. It generates high-quality speech |
|
with features that can be controlled using a simple text prompt (e.g. gender, background noise, speaking rate, pitch and reverberation).</p> |
|
|
|
<p>By default, Parler-TTS generates 🎲 random voice. To ensure 🎯 <b> speaker consistency </b> across generations, these checkpoints were also trained on 34 speakers, characterized by name (e.g. Jon, Lea, Gary, Jenna, Mike, Laura). Learn more about this <a href="https://github.com/huggingface/parler-tts/blob/main/INFERENCE.md#speaker-consistency"> here </a>.</p> |
|
|
|
<p>To take advantage of this, simply adapt your text description to specify which speaker to use: `Jon's voice is monotone...`</p> |
|
""" |
|
) |
|
with gr.Row(): |
|
with gr.Column(): |
|
input_text = gr.Textbox(label="Input Text", lines=2, value=default_text, elem_id="input_text") |
|
description = gr.Textbox(label="Description", lines=2, value=default_description, elem_id="input_description") |
|
use_large = gr.Checkbox(value=False, label="Use Large checkpoint", info="Generate with Parler-TTS Large v1 instead of Mini v1 - Better but way slower.") |
|
run_button = gr.Button("Generate Audio", variant="primary") |
|
with gr.Column(): |
|
audio_out = gr.Audio(label="Parler-TTS generation", type="numpy", elem_id="audio_out") |
|
|
|
inputs = [input_text, description, use_large] |
|
outputs = [audio_out] |
|
run_button.click(fn=gen_tts, inputs=inputs, outputs=outputs, queue=True) |
|
gr.Examples(examples=examples, fn=gen_tts, inputs=inputs, outputs=outputs, cache_examples=True) |
|
gr.HTML( |
|
""" |
|
<p>Tips for ensuring good generation: |
|
<ul> |
|
<li>Include the term "very clear audio" to generate the highest quality audio, and "very noisy audio" for high levels of background noise</li> |
|
<li>Punctuation can be used to control the prosody of the generations, e.g. use commas to add small breaks in speech</li> |
|
<li>The remaining speech features (gender, speaking rate, pitch and reverberation) can be controlled directly through the prompt</li> |
|
</ul> |
|
</p> |
|
|
|
<p>Parler-TTS can be much faster. We give some tips on how to generate much more quickly in this <a href="https://github.com/huggingface/parler-tts/blob/main/INFERENCE.md"> inference guide</a>. Think SDPA, torch.compile, batching and streaming!</p> |
|
|
|
<p>If you want to find out more about how this model was trained and even fine-tune it yourself, check-out the |
|
<a href="https://github.com/huggingface/parler-tts"> Parler-TTS</a> repository on GitHub.</p> |
|
|
|
<p>The Parler-TTS codebase and its associated checkpoints are licensed under <a href='https://github.com/huggingface/parler-tts?tab=Apache-2.0-1-ov-file#readme'> Apache 2.0</a>.</p> |
|
""" |
|
) |
|
|
|
|
|
block.queue() |
|
block.launch(share=True) |
