Build an AI Voice Agent for Automotive with VideoSDK

1sequenceDiagram
2    participant User
3    participant Agent
4    participant VAD
5    participant TurnDetector
6    participant STT
7    participant LLM
8    participant TTS
9    User->>VAD: Speaks (audio input)
10    VAD->>TurnDetector: Detects end of user speech
11    TurnDetector->>STT: Sends audio for transcription
12    STT->>LLM: Transcribed text
13    LLM->>Agent: Generates response
14    Agent->>TTS: Sends response text
15    TTS->>User: Plays synthesized speech
16

1python3.11 -m venv venv
2source venv/bin/activate  # On Windows: venv\\Scripts\\activate
3

1pip install videosdk-agents videosdk-plugins-deepgram videosdk-plugins-openai videosdk-plugins-elevenlabs videosdk-plugins-silero videosdk-plugins-turn-detector
2

1VIDEOSDK_API_KEY=your_videosdk_api_key
2DEEPGRAM_API_KEY=your_deepgram_api_key
3OPENAI_API_KEY=your_openai_api_key
4ELEVENLABS_API_KEY=your_elevenlabs_api_key
5

1import asyncio, os
2from videosdk.agents import Agent, AgentSession, CascadingPipeline, JobContext, RoomOptions, WorkerJob, ConversationFlow
3from videosdk.plugins.silero import SileroVAD
4from videosdk.plugins.turn_detector import TurnDetector, pre_download_model
5from videosdk.plugins.deepgram import DeepgramSTT
6from videosdk.plugins.openai import OpenAILLM
7from videosdk.plugins.elevenlabs import ElevenLabsTTS
8from typing import AsyncIterator
9
10# Pre-downloading the Turn Detector model
11pre_download_model()
12
13agent_instructions = "You are an AI Voice Agent specialized for the automotive industry. Your persona is that of a knowledgeable, friendly, and efficient automotive assistant. Your primary capabilities include: answering questions about vehicle features, maintenance schedules, troubleshooting common car issues, providing navigation assistance, and helping users schedule service appointments. You can also offer information about connected car technologies, safety features, and infotainment systems. You must always prioritize user safety and privacy, never provide driving advice that could endanger users, and refrain from diagnosing complex mechanical issues—always recommend consulting a certified automotive technician for such cases. Do not store or share any personal or vehicle data. If asked for legal, financial, or emergency advice, politely decline and direct the user to the appropriate professional or emergency service. Always communicate clearly and concisely, ensuring instructions are easy to follow, especially when the user may be driving."
14
15class MyVoiceAgent(Agent):
16    def __init__(self):
17        super().__init__(instructions=agent_instructions)
18    async def on_enter(self): await self.session.say("Hello! How can I help?")
19    async def on_exit(self): await self.session.say("Goodbye!")
20
21async def start_session(context: JobContext):
22    # Create agent and conversation flow
23    agent = MyVoiceAgent()
24    conversation_flow = ConversationFlow(agent)
25
26    # Create pipeline
27    pipeline = CascadingPipeline(
28        stt=DeepgramSTT(model="nova-2", language="en"),
29        llm=OpenAILLM(model="gpt-4o"),
30        tts=ElevenLabsTTS(model="eleven_flash_v2_5"),
31        vad=SileroVAD(threshold=0.35),
32        turn_detector=TurnDetector(threshold=0.8)
33    )
34
35    session = AgentSession(
36        agent=agent,
37        pipeline=pipeline,
38        conversation_flow=conversation_flow
39    )
40
41    try:
42        await context.connect()
43        await session.start()
44        # Keep the session running until manually terminated
45        await asyncio.Event().wait()
46    finally:
47        # Clean up resources when done
48        await session.close()
49        await context.shutdown()
50
51def make_context() -> JobContext:
52    room_options = RoomOptions(
53    #  room_id="YOUR_MEETING_ID",  # Set to join a pre-created room; omit to auto-create
54        name="VideoSDK Cascaded Agent",
55        playground=True
56    )
57
58    return JobContext(room_options=room_options)
59
60if __name__ == "__main__":
61    job = WorkerJob(entrypoint=start_session, jobctx=make_context)
62    job.start()
63

1curl -X POST \
2  -H "Authorization: YOUR_VIDEOSDK_API_KEY" \
3  -H "Content-Type: application/json" \
4  -d '{}' \
5  https://api.videosdk.live/v2/rooms
6

1agent_instructions = "You are an AI Voice Agent specialized for the automotive industry. Your persona is that of a knowledgeable, friendly, and efficient automotive assistant. ..."
2
3class MyVoiceAgent(Agent):
4    def __init__(self):
5        super().__init__(instructions=agent_instructions)
6    async def on_enter(self):
7        await self.session.say("Hello! How can I help?")
8    async def on_exit(self):
9        await self.session.say("Goodbye!")
10

1pipeline = CascadingPipeline(
2    stt=DeepgramSTT(model="nova-2", language="en"),
3    llm=OpenAILLM(model="gpt-4o"),
4    tts=ElevenLabsTTS(model="eleven_flash_v2_5"),
5    vad=SileroVAD(threshold=0.35),
6    turn_detector=TurnDetector(threshold=0.8)
7)
8

1async def start_session(context: JobContext):
2    agent = MyVoiceAgent()
3    conversation_flow = ConversationFlow(agent)
4    pipeline = CascadingPipeline(
5        stt=DeepgramSTT(model="nova-2", language="en"),
6        llm=OpenAILLM(model="gpt-4o"),
7        tts=ElevenLabsTTS(model="eleven_flash_v2_5"),
8        vad=SileroVAD(threshold=0.35),
9        turn_detector=TurnDetector(threshold=0.8)
10    )
11    session = AgentSession(agent=agent, pipeline=pipeline, conversation_flow=conversation_flow)
12    try:
13        await context.connect()
14        await session.start()
15        await asyncio.Event().wait()
16    finally:
17        await session.close()
18        await context.shutdown()
19
20def make_context() -> JobContext:
21    room_options = RoomOptions(
22        name="VideoSDK Cascaded Agent",
23        playground=True
24    )
25    return JobContext(room_options=room_options)
26
27if __name__ == "__main__":
28    job = WorkerJob(entrypoint=start_session, jobctx=make_context)
29    job.start()
30

1python main.py
2