Expressive Speech Synthesis with AI Voice Agents

1python -m venv venv
2source venv/bin/activate  # On Windows use `venv\Scripts\activate`
3

1pip install videosdk
2

1VIDEOSDK_API_KEY=your_api_key_here
2

1import asyncio, os
2from videosdk.agents import Agent, AgentSession, CascadingPipeline, JobContext, RoomOptions, WorkerJob, ConversationFlow
3from videosdk.plugins.silero import SileroVAD
4from videosdk.plugins.turn_detector import TurnDetector, pre_download_model
5from videosdk.plugins.deepgram import DeepgramSTT
6from videosdk.plugins.openai import OpenAILLM
7from videosdk.plugins.elevenlabs import ElevenLabsTTS
8from typing import AsyncIterator
9
10# Pre-downloading the Turn Detector model
11pre_download_model()
12
13agent_instructions = "You are an AI Voice Agent specializing in expressive speech synthesis, designed to provide engaging and natural-sounding interactions. Your persona is that of a friendly and knowledgeable virtual assistant who can assist users in various domains by delivering information in a clear and expressive manner. Your capabilities include:
14
151. Utilizing expressive speech synthesis to enhance user engagement and understanding.
162. Answering general knowledge questions across a wide range of topics.
173. Providing step-by-step guidance and instructions in an engaging way.
184. Offering personalized recommendations based on user preferences and history.
19
20Constraints and limitations:
21
221. You are not a subject matter expert in specialized fields such as medicine, law, or finance. Always advise users to consult a professional for expert advice.
232. You must respect user privacy and confidentiality, ensuring that no personal data is stored or shared without consent.
243. You should avoid making any promises or guarantees about outcomes or results.
254. Your responses should be concise and to the point, avoiding overly technical jargon unless specifically requested by the user."
26
27class MyVoiceAgent(Agent):
28    def __init__(self):
29        super().__init__(instructions=agent_instructions)
30    async def on_enter(self): await self.session.say("Hello! How can I help?")
31    async def on_exit(self): await self.session.say("Goodbye!")
32
33async def start_session(context: JobContext):
34    # Create agent and conversation flow
35    agent = MyVoiceAgent()
36    conversation_flow = ConversationFlow(agent)
37
38    # Create pipeline
39    pipeline = CascadingPipeline(
40        stt=DeepgramSTT(model="nova-2", language="en"),
41        llm=OpenAILLM(model="gpt-4o"),
42        tts=ElevenLabsTTS(model="eleven_flash_v2_5"),
43        vad=SileroVAD(threshold=0.35),
44        turn_detector=TurnDetector(threshold=0.8)
45    )
46
47    session = [AgentSession](https://docs.videosdk.live/ai_agents/core-components/agent-session)(
48        agent=agent,
49        pipeline=pipeline,
50        conversation_flow=conversation_flow
51    )
52
53    try:
54        await context.connect()
55        await session.start()
56        # Keep the session running until manually terminated
57        await asyncio.Event().wait()
58    finally:
59        # Clean up resources when done
60        await session.close()
61        await context.shutdown()
62
63def make_context() -> JobContext:
64    room_options = RoomOptions(
65    #  room_id="YOUR_MEETING_ID",  # Set to join a pre-created room; omit to auto-create
66        name="VideoSDK Cascaded Agent",
67        playground=True
68    )
69
70    return JobContext(room_options=room_options)
71
72if __name__ == "__main__":
73    job = WorkerJob(entrypoint=start_session, jobctx=make_context)
74    job.start()
75

1curl -X POST "https://api.videosdk.live/v1/meetings" \
2-H "Authorization: Bearer YOUR_API_KEY" \
3-H "Content-Type: application/json"
4

1python main.py
2