Build a Healthcare AI Voice Agent with VideoSDK

1sequenceDiagram
2  participant User
3  participant Microphone
4  participant Agent
5  participant STT
6  participant LLM
7  participant TTS
8  participant Speaker
9  User->>Microphone: Speaks
10  Microphone->>Agent: Audio Stream
11  Agent->>STT: Audio for Transcription
12  STT->>Agent: Text
13  Agent->>LLM: Text for Response
14  LLM->>Agent: Generated Reply
15  Agent->>TTS: Text for Synthesis
16  TTS->>Agent: Audio
17  Agent->>Speaker: Plays Response
18  Speaker->>User: Hears Reply
19

1python -m venv venv
2# On Windows:
3venv\Scripts\activate
4# On macOS/Linux:
5source venv/bin/activate
6

1pip install videosdk-agents videosdk-plugin-deepgram videosdk-plugin-openai videosdk-plugin-elevenlabs videosdk-plugin-silero videosdk-plugin-turn-detector
2

1DEEPGRAM_API_KEY=your_deepgram_api_key
2OPENAI_API_KEY=your_openai_api_key
3ELEVENLABS_API_KEY=your_elevenlabs_api_key
4VIDEOSDK_API_KEY=your_videosdk_api_key
5

1import asyncio, os
2from videosdk.agents import Agent, AgentSession, CascadingPipeline, JobContext, RoomOptions, WorkerJob, ConversationFlow
3from videosdk.plugins.silero import SileroVAD
4from videosdk.plugins.turn_detector import TurnDetector, pre_download_model
5from videosdk.plugins.deepgram import DeepgramSTT
6from videosdk.plugins.openai import OpenAILLM
7from videosdk.plugins.elevenlabs import ElevenLabsTTS
8from typing import AsyncIterator
9
10# Pre-downloading the Turn Detector model
11pre_download_model()
12
13agent_instructions = "You are a helpful healthcare AI voice agent designed to assist users with general healthcare inquiries and administrative tasks. Your persona is empathetic, patient, and professional, aiming to provide clear and supportive guidance. Your capabilities include: answering general questions about common symptoms, providing information on healthcare services, assisting with appointment scheduling, offering reminders for medications or check-ups, and helping users navigate healthcare resources. Constraints: You are not a licensed medical professional and cannot diagnose, prescribe, or provide medical advice. Always include a disclaimer that users should consult a qualified healthcare provider for medical concerns. Do not collect or store any sensitive personal health information. Maintain user privacy and adhere to HIPAA guidelines where applicable. If a user requests urgent medical help, instruct them to contact emergency services immediately."
14
15class MyVoiceAgent(Agent):
16    def __init__(self):
17        super().__init__(instructions=agent_instructions)
18    async def on_enter(self): await self.session.say("Hello! How can I help?")
19    async def on_exit(self): await self.session.say("Goodbye!")
20
21async def start_session(context: JobContext):
22    # Create agent and conversation flow
23    agent = MyVoiceAgent()
24    conversation_flow = ConversationFlow(agent)
25
26    # Create pipeline
27    pipeline = CascadingPipeline(
28        stt=DeepgramSTT(model="nova-2", language="en"),
29        llm=OpenAILLM(model="gpt-4o"),
30        tts=ElevenLabsTTS(model="eleven_flash_v2_5"),
31        vad=SileroVAD(threshold=0.35),
32        turn_detector=TurnDetector(threshold=0.8)
33    )
34
35    session = AgentSession(
36        agent=agent,
37        pipeline=pipeline,
38        conversation_flow=conversation_flow
39    )
40
41    try:
42        await context.connect()
43        await session.start()
44        # Keep the session running until manually terminated
45        await asyncio.Event().wait()
46    finally:
47        # Clean up resources when done
48        await session.close()
49        await context.shutdown()
50
51def make_context() -> JobContext:
52    room_options = RoomOptions(
53    #  room_id="YOUR_MEETING_ID",  # Set to join a pre-created room; omit to auto-create
54        name="VideoSDK Cascaded Agent",
55        playground=True
56    )
57
58    return JobContext(room_options=room_options)
59
60if __name__ == "__main__":
61    job = WorkerJob(entrypoint=start_session, jobctx=make_context)
62    job.start()
63

1curl -X POST -H "Authorization: YOUR_VIDEOSDK_API_KEY" -H "Content-Type: application/json" -d '{}' https://api.videosdk.live/v2/rooms
2

1agent_instructions = "You are a helpful healthcare AI voice agent designed to assist users with general healthcare inquiries and administrative tasks. ..."
2
3class MyVoiceAgent(Agent):
4    def __init__(self):
5        super().__init__(instructions=agent_instructions)
6    async def on_enter(self):
7        await self.session.say("Hello! How can I help?")
8    async def on_exit(self):
9        await self.session.say("Goodbye!")
10

1pipeline = CascadingPipeline(
2    stt=DeepgramSTT(model="nova-2", language="en"),
3    llm=OpenAILLM(model="gpt-4o"),
4    tts=ElevenLabsTTS(model="eleven_flash_v2_5"),
5    vad=SileroVAD(threshold=0.35),
6    turn_detector=TurnDetector(threshold=0.8)
7)
8

1async def start_session(context: JobContext):
2    agent = MyVoiceAgent()
3    conversation_flow = ConversationFlow(agent)
4    pipeline = CascadingPipeline(...)
5    session = AgentSession(agent=agent, pipeline=pipeline, conversation_flow=conversation_flow)
6    try:
7        await context.connect()
8        await session.start()
9        await asyncio.Event().wait()
10    finally:
11        await session.close()
12        await context.shutdown()
13
14def make_context() -> JobContext:
15    room_options = RoomOptions(
16        name="VideoSDK Cascaded Agent",
17        playground=True
18    )
19    return JobContext(room_options=room_options)
20
21if __name__ == "__main__":
22    job = WorkerJob(entrypoint=start_session, jobctx=make_context)
23    job.start()
24

1python main.py
2