Spaces:
Runtime error
Runtime error
File size: 3,045 Bytes
499588c de1d53c e715e06 62144b2 e715e06 adc0c37 e715e06 3ad82db adc0c37 3ad82db adc0c37 e715e06 931ad53 3d8470f 62144b2 3d8470f 3501caf 62144b2 86df8f3 62144b2 e715e06 62144b2 4fa3f9a adc0c37 e715e06 956d8c0 e715e06 956d8c0 e715e06 24baf06 e715e06 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 |
import os
import spaces
import gradio as gr
import librosa
import numpy as np
from speechbrain.inference import EncoderClassifier
import torch
from transformers import SpeechT5ForTextToSpeech, SpeechT5Processor, SpeechT5HifiGan
checkpoint = "techiaith/microsoft_speecht5_finetuned_bu_tts_cy_en"
processor = SpeechT5Processor.from_pretrained(checkpoint)
model = SpeechT5ForTextToSpeech.from_pretrained(checkpoint)
vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
speaker_embeddings = {
"GGP": "spkemb/speaker0.npy",
"BGP": "spkemb/speaker1.npy",
"BDP": "spkemb/speaker2.npy",
}
spk_model_name = "speechbrain/spkrec-xvect-voxceleb"
device = "cuda" if torch.cuda.is_available() else "cpu"
speaker_model = EncoderClassifier.from_hparams(
source=spk_model_name,
run_opts={"device": device},
savedir=os.path.join("/tmp", spk_model_name),
)
def create_speaker_embedding(waveform):
with torch.no_grad():
se = speaker_model.encode_batch(torch.tensor(waveform))
se = torch.nn.functional.normalize(se, dim=2)
se = se.squeeze().cpu().numpy()
return se
@spaces.GPU
def predict(text, speaker, audio):
if len(text.strip()) == 0:
return (16000, np.zeros(0).astype(np.int16))
if audio is not None:
speaker_embedding = create_speaker_embedding(audio)
else:
speaker_embedding = np.load(speaker_embeddings[speaker[:3]])
speaker_embedding = torch.tensor(speaker_embedding).unsqueeze(0)
inputs = processor(text=text, return_tensors="pt")
speech = model.generate_speech(inputs["input_ids"], speaker_embedding, vocoder=vocoder)
speech = (speech.numpy() * 32767).astype(np.int16)
return (16000, speech)
title = "Techiaith Finetune Microsoft/SpeechT5: Speech Synthesis"
description = """
Lleisiau TTS microsoft_speech_T5_finetune_bu_tts_cy_en
"""
examples = [
["Rhyfeddod neu ffenomenon optegol a meteorolegol yw enfys, pan fydd sbectrwm o olau yn ymddangos yn yr awyr pan fo'r haul yn disgleirio ar ddiferion o leithder yn atmosffer y ddaear.", "GGP (gwryw-gogledd-pro)"],
["Rhyfeddod neu ffenomenon optegol a meteorolegol yw enfys, pan fydd sbectrwm o olau yn ymddangos yn yr awyr pan fo'r haul yn disgleirio ar ddiferion o leithder yn atmosffer y ddaear.", "BGP (benyw-gogledd-pro)"],
["Rhyfeddod neu ffenomenon optegol a meteorolegol yw enfys, pan fydd sbectrwm o olau yn ymddangos yn yr awyr pan fo'r haul yn disgleirio ar ddiferion o leithder yn atmosffer y ddaear.", "BDP (benyw-de-pro)"],
]
gr.Interface(
fn=predict,
inputs=[
gr.Text(label="Input Text"),
gr.Audio(sources="microphone", type="filepath"),
gr.Radio(label="Speaker", choices=[
"GGP (gwryw-gogledd-pro)",
"BGP (benyw-gogledd-pro)",
"BDP (benyw-de-pro)",
],
value="GGP (gwryw-gogledd-pro)"),
],
outputs=[
gr.Audio(label="Generated Speech", type="numpy"),
],
title=title,
description=description,
examples=examples,
).launch()
|