File size: 3,199 Bytes
d347764 b239645 d347764 b239645 d347764 b239645 d347764 b239645 d347764 b239645 950d1a2 d347764 b239645 d347764 b239645 c949774 7b9bf4b c949774 d347764 b239645 c949774 d347764 f805e49 c6f1d54 f805e49 b239645 f805e49 b239645 c737803 d347764 226ec3a d347764 f805e49 d347764 c737803 b239645 c737803 b239645 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 |
import gradio as gr
import numpy as np
import torch
from datasets import load_dataset
from transformers import pipeline, AutoTokenizer, VitsModel, VitsTokenizer
device = "cuda:0" if torch.cuda.is_available() else "cpu"
# load speech translation checkpoint
asr_pipe = pipeline("automatic-speech-recognition",
model="openai/whisper-base", device=device)
# load text-to-speech checkpoint and speaker embeddings
# processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
# model = SpeechT5ForTextToSpeech.from_pretrained(
# "microsoft/speecht5_tts").to(device)
# vocoder = SpeechT5HifiGan.from_pretrained(
# "microsoft/speecht5_hifigan").to(device)
# embeddings_dataset = load_dataset(
# "Matthijs/cmu-arctic-xvectors", split="validation")
# speaker_embeddings = torch.tensor(
# embeddings_dataset[7306]["xvector"]).unsqueeze(0)
model_nld = VitsModel.from_pretrained("facebook/mms-tts-ara")
tokenizer = VitsTokenizer.from_pretrained("facebook/mms-tts-ara")
def translate(audio):
outputs = asr_pipe(audio, max_new_tokens=256,
generate_kwargs={"task": "translate"})
return outputs["text"]
def synthesise2(text):
# inputs = processor(text=text, return_tensors="pt")
# speech = model.generate_speech(inputs["input_ids"].to(device), speaker_embeddings.to(device), vocoder=vocoder)
inputs = tokenizer(text, return_tensors="pt")
with torch.no_grad():
outputs = model_nld(**inputs)
#speech = outputs.waveform[0]
audio_tensor = outputs.audio
# Convert tensor to numpy array
audio_array = audio_tensor.numpy()
# Reshape the audio array to 1D
audio_array = audio_array.reshape(-1)
return audio_array
def speech_to_speech_translation(audio):
translated_text = translate(audio)
synthesised_speech = synthesise2(translated_text)
synthesised_speech = np.int16(synthesised_speech * 32767)
return 16000, synthesised_speech
title = "Cascaded STST"
description = """
Demo for cascaded speech-to-speech translation (STST), mapping from source speech in any language to target speech in English. Demo uses OpenAI's [Whisper Base](https://huggingface.co/openai/whisper-base) model for speech translation, and Microsoft's
[SpeechT5 TTS](https://huggingface.co/microsoft/speecht5_tts) model for text-to-speech:
"""
demo = gr.Blocks()
mic_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="microphone", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
title=title,
description=description,
)
file_translate = gr.Interface(
fn=speech_to_speech_translation,
inputs=gr.Audio(source="upload", type="filepath"),
outputs=gr.Audio(label="Generated Speech", type="numpy"),
examples=[["./example.wav"]],
title=title,
description=description,
)
with demo:
gr.TabbedInterface([mic_translate, file_translate],
["Microphone", "Audio File"])
demo.launch() |