Code

.gitignore

@@ -0,0 +1 @@
+.env

app.py

@@ -0,0 +1,141 @@
+import gradio as gr
+from gradio_webrtc import WebRTC, ReplyOnPause, AdditionalOutputs
+import anthropic
+from pyht import Client as PyHtClient, TTSOptions
+import dataclasses
+import os
+import numpy as np
+from huggingface_hub import InferenceClient
+import io
+from pydub import AudioSegment
+from dotenv import load_dotenv
+
+load_dotenv()
+
+account_sid = os.environ.get("TWILIO_ACCOUNT_SID")
+auth_token = os.environ.get("TWILIO_AUTH_TOKEN")
+
+if account_sid and auth_token:
+    from twilio.rest import Client
+    client = Client(account_sid, auth_token)
+
+    token = client.tokens.create()
+
+    rtc_configuration = {
+        "iceServers": token.ice_servers,
+        "iceTransportPolicy": "relay",
+    }
+else:
+    rtc_configuration = None
+
+
+@dataclasses.dataclass
+class Clients:
+    claude: anthropic.Anthropic
+    play_ht: PyHtClient
+    hf: InferenceClient
+
+
+tts_options = TTSOptions(voice="s3://voice-cloning-zero-shot/775ae416-49bb-4fb6-bd45-740f205d20a1/jennifersaad/manifest.json",
+                         sample_rate=24000)
+
+
+def aggregate_chunks(chunks_iterator):
+    leftover = b''  # Store incomplete bytes between chunks
+    
+    for chunk in chunks_iterator:
+        # Combine with any leftover bytes from previous chunk
+        current_bytes = leftover + chunk
+        
+        # Calculate complete samples
+        n_complete_samples = len(current_bytes) // 2  # int16 = 2 bytes
+        bytes_to_process = n_complete_samples * 2
+        
+        # Split into complete samples and leftover
+        to_process = current_bytes[:bytes_to_process]
+        leftover = current_bytes[bytes_to_process:]
+        
+        if to_process:  # Only yield if we have complete samples
+            audio_array = np.frombuffer(to_process, dtype=np.int16).reshape(1, -1)
+            yield audio_array
+
+
+def audio_to_bytes(audio: tuple[int, np.ndarray]) -> bytes:
+    audio_buffer = io.BytesIO()
+    segment = AudioSegment(
+        audio[1].tobytes(),
+        frame_rate=audio[0],
+        sample_width=audio[1].dtype.itemsize,
+        channels=1,
+    )
+    segment.export(audio_buffer, format="mp3")
+    return audio_buffer.getvalue()
+
+
+def set_api_key(claude_key, play_ht_username, play_ht_key):
+    claude_client = anthropic.Anthropic(api_key=claude_key)
+    play_ht_client = PyHtClient(user_id=play_ht_username, api_key=play_ht_key)
+    gr.Info("Successfully set API keys.", duration=3)
+    return Clients(claude=claude_client, play_ht=play_ht_client,
+                   hf=InferenceClient()), gr.skip()
+
+
+def response(audio: tuple[int, np.ndarray], conversation_llm_format: list[dict],
+             chatbot: list[dict], client_state: Clients):
+    if not client_state:
+        raise gr.Error("Please set your API keys first.")
+    prompt = client_state.hf.automatic_speech_recognition(audio_to_bytes(audio)).text
+    conversation_llm_format.append({"role": "user", "content": prompt})
+    response = client_state.claude.messages.create(
+        model="claude-3-5-haiku-20241022",
+        max_tokens=512,
+        messages=conversation_llm_format,
+    )
+    response_text = " ".join(block.text for block in response.content if getattr(block, "type", None) == "text")
+    conversation_llm_format.append({"role": "assistant", "content": response_text})
+    chatbot.append({"role": "user", "content": prompt})
+    chatbot.append({"role": "assistant", "content": response_text})
+    yield AdditionalOutputs(conversation_llm_format, chatbot)
+    iterator = client_state.play_ht.tts(response_text, options=tts_options, voice_engine="Play3.0")
+    for chunk in aggregate_chunks(iterator):
+        audio_array = np.frombuffer(chunk, dtype=np.int16).reshape(1, -1)
+        yield (24000, audio_array, "mono")
+
+
+with gr.Blocks() as demo:
+    with gr.Group():
+        with gr.Row():
+            chatbot = gr.Chatbot(label="Conversation", type="messages")
+        with gr.Row(equal_height=True):
+            with gr.Column(scale=1):
+                with gr.Row():
+                    claude_key = gr.Textbox(type="password", value=os.getenv("ANTHROPIC_API_KEY"),
+                                            label="Enter your Anthropic API Key")
+                    play_ht_username = gr.Textbox(type="password",
+                                                value=os.getenv("PLAY_HT_USER_ID"),
+                                                label="Enter your PlayHt Username")
+                    play_ht_key = gr.Textbox(type="password",
+                                            value=os.getenv("PLAY_HT_API_KEY"),
+                                            label="Enter your PlayHt API Key")
+                with gr.Row():
+                    set_key_button = gr.Button("Set Keys", variant="primary")
+            with gr.Column(scale=5):
+                audio = WebRTC(modality="audio", mode="send-receive",
+                                label="Audio Stream",
+                                rtc_configuration=rtc_configuration)
+
+    client_state = gr.State(None)
+    conversation_llm_format = gr.State([])
+
+    set_key_button.click(set_api_key, inputs=[claude_key, play_ht_username, play_ht_key],
+                         outputs=[client_state, set_key_button])
+    audio.stream(
+        ReplyOnPause(response),
+        inputs=[audio, conversation_llm_format, chatbot, client_state],
+        outputs=[audio]
+    )
+    audio.on_additional_outputs(lambda l, g: (l, g), outputs=[conversation_llm_format, chatbot])
+
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,5 @@
+gradio_webrtc[vad]==0.0.11
+anthropic
+pyht
+twilio
+python-dotenv