tts

app.py

@@ -0,0 +1,278 @@
+import gradio as gr
+import numpy as np
+import torch
+from moshi.models.loaders import CheckpointInfo
+from moshi.models.tts import DEFAULT_DSM_TTS_REPO, TTSModel
+
+checkpoint_info = CheckpointInfo.from_hf_repo(DEFAULT_DSM_TTS_REPO)
+tts_model = TTSModel.from_checkpoint_info(
+    checkpoint_info, n_q=32, temp=0.6, device=torch.device("cpu")
+)
+
+
+def generate_audio(text, voice):
+    entries = tts_model.prepare_script([text], padding_between=1)
+    voice_path = tts_model.get_voice_path(voice)
+    condition_attributes = tts_model.make_condition_attributes(
+        [voice_path], cfg_coef=2.0
+    )
+
+    pcms = []
+
+    def _on_frame(frame):
+        if (frame != -1).all():
+            pcm = tts_model.mimi.decode(frame[:, 1:, :]).cpu().numpy()
+            pcms.append(np.clip(pcm[0, 0], -1, 1))
+
+    all_entries = [entries]
+    all_condition_attributes = [condition_attributes]
+
+    with tts_model.mimi.streaming(len(all_entries)):
+        tts_model.generate(all_entries, all_condition_attributes, on_frame=_on_frame)
+
+    if len(pcms) == 0:
+        return None
+
+    audio = np.concatenate(pcms, axis=-1)
+    sample_rate = tts_model.mimi.sample_rate
+    return (sample_rate, audio)
+
+
+iface = gr.Interface(
+    fn=generate_audio,
+    inputs=[
+        gr.Textbox(
+            label="Input Text",
+            placeholder="Enter text to convert to speech...",
+            lines=3,
+            value="This is a test of the text-to-speech model. Please select a voice and click 'Submit' to generate audio.",
+        ),
+        gr.Dropdown(
+            label="Voice",
+            choices=[
+                "unmute-prod-website/default_voice.wav",
+                "unmute-prod-website/p329_022.wav",
+                "unmute-prod-website/degaulle-2.wav",
+                "unmute-prod-website/fabieng-enhanced-v2.wav",
+                "unmute-prod-website/ex04_narration_longform_00001.wav",
+                "unmute-prod-website/developpeuse-3.wav",
+                "expresso/ex04-ex02_nonverbal_004_channel2_71s.wav",
+                "expresso/ex01-ex02_projected_002_channel2_248s.wav",
+                "expresso/ex03-ex01_nonverbal_001_channel2_37s.wav",
+                "expresso/ex03-ex01_angry_001_channel1_201s.wav",
+                "expresso/ex01-ex02_enunciated_001_channel2_354s.wav",
+                "expresso/ex03-ex02_laughing_001_channel1_248s.wav",
+                "expresso/ex01-ex02_enunciated_001_channel1_432s.wav",
+                "expresso/ex04-ex01_disgusted_001_channel1_130s.wav",
+                "expresso/ex01-ex02_whisper_001_channel2_717s.wav",
+                "expresso/ex04-ex02_confused_001_channel2_488s.wav",
+                "expresso/ex03-ex02_animal-animaldir_003_channel1_32s.wav",
+                "expresso/ex04-ex01_narration_001_channel2_686s.wav",
+                "expresso/ex01-ex02_fast_001_channel1_104s.wav",
+                "expresso/ex01-ex02_fast_001_channel2_73s.wav",
+                "expresso/ex03-ex02_animaldir-animal_008_channel1_147s.wav",
+                "expresso/ex04-ex02_happy_001_channel1_118s.wav",
+                "expresso/ex04-ex02_enunciated_001_channel2_898s.wav",
+                "expresso/ex03-ex01_calm_001_channel2_1081s.wav",
+                "expresso/ex04-ex03_fast_001_channel2_25s.wav",
+                "expresso/ex03-ex01_laughing_002_channel2_232s.wav",
+                "expresso/ex03-ex01_disgusted_004_channel1_170s.wav",
+                "expresso/ex01-ex02_default_001_channel1_168s.wav",
+                "expresso/ex04-ex01_animal-animaldir_006_channel1_196s.wav",
+                "expresso/ex01-ex02_default_001_channel2_198s.wav",
+                "expresso/ex03-ex02_childdir-child_004_channel2_187s.wav",
+                "expresso/ex03-ex01_sarcastic_001_channel2_491s.wav",
+                "expresso/ex03-ex01_confused_001_channel2_816s.wav",
+                "expresso/ex03-ex02_sympathetic-sad_008_channel1_215s.wav",
+                "expresso/ex04-ex02_sarcastic_001_channel1_519s.wav",
+                "expresso/ex03-ex01_calm_001_channel1_1143s.wav",
+                "expresso/ex04-ex01_child-childdir_003_channel2_283s.wav",
+                "expresso/ex03-ex01_sleepy_001_channel1_619s.wav",
+                "expresso/ex04-ex02_enunciated_001_channel1_496s.wav",
+                "expresso/ex03-ex01_desire_004_channel1_545s.wav",
+                "expresso/ex04-ex01_laughing_001_channel1_306s.wav",
+                "expresso/ex03-ex01_happy_001_channel2_257s.wav",
+                "expresso/ex04-ex01_disgusted_001_channel2_325s.wav",
+                "expresso/ex04-ex02_bored_001_channel1_254s.wav",
+                "expresso/ex04-ex02_awe_001_channel1_982s.wav",
+                "expresso/ex04-ex01_sympathetic-sad_008_channel2_453s.wav",
+                "expresso/ex04-ex03_enunciated_001_channel1_86s.wav",
+                "expresso/ex01-ex02_whisper_001_channel1_579s.wav",
+                "expresso/ex03-ex01_enunciated_001_channel1_388s.wav",
+                "expresso/ex04-ex03_whisper_002_channel2_266s.wav",
+                "expresso/ex04-ex03_default_001_channel1_3s.wav",
+                "expresso/ex04-ex02_calm_002_channel1_480s.wav",
+                "expresso/ex04-ex03_whisper_001_channel1_198s.wav",
+                "expresso/ex04-ex01_laughing_001_channel2_293s.wav",
+                "expresso/ex03-ex01_nonverbal_006_channel1_62s.wav",
+                "expresso/ex04-ex02_happy_001_channel2_140s.wav",
+                "expresso/ex04-ex02_desire_001_channel2_694s.wav",
+                "expresso/ex03-ex02_narration_002_channel2_1136s.wav",
+                "expresso/ex04-ex01_animaldir-animal_001_channel1_118s.wav",
+                "expresso/ex04-ex03_projected_001_channel2_179s.wav",
+                "expresso/ex03-ex01_laughing_001_channel1_188s.wav",
+                "expresso/ex04-ex02_laughing_001_channel1_147s.wav",
+                "expresso/ex03-ex01_desire_004_channel2_580s.wav",
+                "expresso/ex04-ex02_fearful_001_channel2_266s.wav",
+                "expresso/ex04-ex01_narration_001_channel1_605s.wav",
+                "expresso/ex03-ex01_awe_001_channel1_1323s.wav",
+                "expresso/ex04-ex01_childdir-child_001_channel1_228s.wav",
+                "expresso/ex04-ex01_animaldir-animal_004_channel2_88s.wav",
+                "expresso/ex04-ex01_animal-animaldir_006_channel2_49s.wav",
+                "expresso/ex03-ex01_happy_001_channel1_334s.wav",
+                "expresso/ex03-ex02_sad-sympathetic_001_channel1_454s.wav",
+                "expresso/ex03-ex02_sympathetic-sad_008_channel2_268s.wav",
+                "expresso/ex04-ex02_bored_001_channel2_232s.wav",
+                "expresso/ex04-ex03_projected_001_channel1_192s.wav",
+                "expresso/ex04-ex03_enunciated_001_channel2_342s.wav",
+                "expresso/ex04-ex01_sympathetic-sad_008_channel1_415s.wav",
+                "expresso/ex04-ex02_laughing_001_channel2_159s.wav",
+                "expresso/ex04-ex02_calm_001_channel2_336s.wav",
+                "expresso/ex04-ex02_awe_001_channel2_1013s.wav",
+                "expresso/ex03-ex02_narration_001_channel1_674s.wav",
+                "expresso/ex03-ex01_awe_001_channel2_1290s.wav",
+                "expresso/ex03-ex01_sarcastic_001_channel1_435s.wav",
+                "expresso/ex04-ex02_disgusted_004_channel1_169s.wav",
+                "expresso/ex04-ex02_desire_001_channel1_657s.wav",
+                "expresso/ex04-ex02_nonverbal_004_channel1_18s.wav",
+                "expresso/ex04-ex03_default_002_channel2_239s.wav",
+                "expresso/ex04-ex02_confused_001_channel1_499s.wav",
+                "expresso/ex01-ex02_projected_001_channel1_46s.wav",
+                "expresso/ex04-ex02_angry_001_channel2_150s.wav",
+                "expresso/ex03-ex02_child-childdir_001_channel2_69s.wav",
+                "expresso/ex03-ex01_confused_001_channel1_909s.wav",
+                "expresso/ex03-ex01_angry_001_channel2_181s.wav",
+                "expresso/ex04-ex03_fast_001_channel1_208s.wav",
+                "expresso/ex04-ex02_fearful_001_channel1_316s.wav",
+                "expresso/ex03-ex02_animaldir-animal_008_channel2_136s.wav",
+                "expresso/ex03-ex02_child-childdir_001_channel1_291s.wav",
+                "expresso/ex03-ex02_laughing_001_channel2_234s.wav",
+                "expresso/ex03-ex02_sad-sympathetic_001_channel2_400s.wav",
+                "expresso/ex04-ex01_sad-sympathetic_001_channel1_267s.wav",
+                "expresso/ex03-ex01_sleepy_001_channel2_662s.wav",
+                "expresso/ex03-ex02_childdir-child_004_channel1_308s.wav",
+                "expresso/ex04-ex01_sad-sympathetic_001_channel2_346s.wav",
+                "expresso/ex04-ex02_disgusted_001_channel2_98s.wav",
+                "expresso/ex03-ex01_enunciated_001_channel2_576s.wav",
+                "expresso/ex04-ex02_sarcastic_001_channel2_466s.wav",
+                "expresso/ex04-ex01_child-childdir_004_channel1_118s.wav",
+                "expresso/ex04-ex02_angry_001_channel1_119s.wav",
+                "expresso/ex03-ex02_animal-animaldir_002_channel2_89s.wav",
+                "expresso/ex04-ex01_childdir-child_001_channel2_420s.wav",
+                "vctk/p323_023.wav",
+                "vctk/p333_023.wav",
+                "vctk/p280_023.wav",
+                "vctk/p239_023.wav",
+                "vctk/p341_023.wav",
+                "vctk/p229_023.wav",
+                "vctk/p351_023.wav",
+                "vctk/p264_023.wav",
+                "vctk/p274_023.wav",
+                "vctk/p299_023.wav",
+                "vctk/p230_023.wav",
+                "vctk/p252_023.wav",
+                "vctk/p305_023.wav",
+                "vctk/p315_023.wav",
+                "vctk/p231_023.wav",
+                "vctk/p288_023.wav",
+                "vctk/p298_023.wav",
+                "vctk/p253_023.wav",
+                "vctk/p243_023.wav",
+                "vctk/p376_023.wav",
+                "vctk/p314_023.wav",
+                "vctk/p304_023.wav",
+                "vctk/p228_023.wav",
+                "vctk/p238_023.wav",
+                "vctk/p281_023.wav",
+                "vctk/p275_023.wav",
+                "vctk/p265_023.wav",
+                "vctk/p339_023.wav",
+                "vctk/p241_023.wav",
+                "vctk/p329_023.wav",
+                "vctk/p251_023.wav",
+                "vctk/p233_023.wav",
+                "vctk/p306_023.wav",
+                "vctk/p316_023.wav",
+                "vctk/p364_023.wav",
+                "vctk/p374_023.wav",
+                "vctk/p283_023.wav",
+                "vctk/p293_023.wav",
+                "vctk/p258_023.wav",
+                "vctk/p330_023.wav",
+                "vctk/p248_023.wav",
+                "vctk/p267_023.wav",
+                "vctk/p277_023.wav",
+                "vctk/p292_023.wav",
+                "vctk/p282_023.wav",
+                "vctk/p343_023.wav",
+                "vctk/p249_023.wav",
+                "vctk/p259_023.wav",
+                "vctk/p276_023.wav",
+                "vctk/p266_023.wav",
+                "vctk/p250_023.wav",
+                "vctk/p240_023.wav",
+                "vctk/p232_023.wav",
+                "vctk/p317_023.wav",
+                "vctk/p307_023.wav",
+                "vctk/p302_023.wav",
+                "vctk/p312_023.wav",
+                "vctk/p360_023.wav",
+                "vctk/p245_023.wav",
+                "vctk/p255_023.wav",
+                "vctk/p227_023.wav",
+                "vctk/p237_023.wav",
+                "vctk/p263_023.wav",
+                "vctk/p273_023.wav",
+                "vctk/p287_023.wav",
+                "vctk/p297_023.wav",
+                "vctk/p334_023.wav",
+                "vctk/s5_023.wav",
+                "vctk/p272_023.wav",
+                "vctk/p262_023.wav",
+                "vctk/p347_023.wav",
+                "vctk/p286_023.wav",
+                "vctk/p335_023.wav",
+                "vctk/p313_023.wav",
+                "vctk/p303_023.wav",
+                "vctk/p361_023.wav",
+                "vctk/p254_023.wav",
+                "vctk/p244_023.wav",
+                "vctk/p236_023.wav",
+                "vctk/p226_023.wav",
+                "vctk/p318_023.wav",
+                "vctk/p260_023.wav",
+                "vctk/p308_023.wav",
+                "vctk/p270_023.wav",
+                "vctk/p345_023.wav",
+                "vctk/p284_023.wav",
+                "vctk/p294_023.wav",
+                "vctk/p363_023.wav",
+                "vctk/p301_023.wav",
+                "vctk/p279_023.wav",
+                "vctk/p311_023.wav",
+                "vctk/p269_023.wav",
+                "vctk/p234_023.wav",
+                "vctk/p246_023.wav",
+                "vctk/p256_023.wav",
+                "vctk/p310_023.wav",
+                "vctk/p278_023.wav",
+                "vctk/p300_023.wav",
+                "vctk/p225_023.wav",
+                "vctk/p257_023.wav",
+                "vctk/p247_023.wav",
+                "vctk/p271_023.wav",
+                "vctk/p261_023.wav",
+                "vctk/p336_023.wav",
+                "vctk/p326_023.wav",
+                "vctk/p285_023.wav",
+            ],
+            value="expresso/ex03-ex01_happy_001_channel1_334s.wav",
+        ),
+    ],
+    outputs=gr.Audio(label="Generated Audio"),
+    title="kyutai/tts-1.6b-en_fr",
+    description="Convert text to speech using the kyutai/tts-1.6b-en_fr model. Select a voice and enter text to generate audio.",
+)
+
+if __name__ == "__main__":
+    iface.launch()

inference.py

@@ -0,0 +1,33 @@
+import argparse
+import numpy as np
+import torch
+from moshi.models.loaders import CheckpointInfo
+from moshi.models.tts import DEFAULT_DSM_TTS_REPO, DEFAULT_DSM_TTS_VOICE_REPO, TTSModel
+
+def load_model():
+    checkpoint_info = CheckpointInfo.from_hf_repo(DEFAULT_DSM_TTS_REPO)
+    tts_model = TTSModel.from_checkpoint_info(
+        checkpoint_info, n_q=32, temp=0.6, device=torch.device("cpu")
+    )
+    return tts_model
+
+def generate_audio(tts_model, text, voice):
+    entries = tts_model.prepare_script([text], padding_between=1)
+    voice_path = tts_model.get_voice_path(voice)
+    condition_attributes = tts_model.make_condition_attributes(
+        [voice_path], cfg_coef=2.0
+    )
+
+    pcms = []
+    def _on_frame(frame):
+        if (frame != -1).all():
+            pcm = tts_model.mimi.decode(frame[:, 1:, :]).cpu().numpy()
+            pcms.append(np.clip(pcm[0, 0], -1, 1))
+
+    all_entries = [entries]
+    all_condition_attributes = [condition_attributes]
+    with tts_model.mimi.streaming(len(all_entries)):
+        tts_model.generate(all_entries, all_condition_attributes, on_frame=_on_frame)
+
+    audio = np.concatenate(pcms, axis=-1)
+    return audio

utils.py

@@ -0,0 +1,20 @@
+import os
+
+def load_voice_path(voice):
+    """Load the voice path from the given voice identifier."""
+    return os.path.join("path_to_voice_repo", voice)
+
+def load_configuration(config_file):
+    """Load configuration settings from a specified file."""
+    import json
+    with open(config_file, 'r') as f:
+        return json.load(f)
+
+def save_audio_file(audio, file_path):
+    """Save the generated audio to a specified file path."""
+    from scipy.io.wavfile import write
+    write(file_path, 22050, audio)  # Assuming a sample rate of 22050 Hz
+
+def clip_audio(audio):
+    """Clip audio to the range [-1, 1]."""
+    return np.clip(audio, -1, 1)