How to Build a Voice AI Call Center Agent (Step-by-Step Guide)

1python3 -m venv venv
2source venv/bin/activate  # On Windows: venv\Scripts\activate
3

1pip install videosdk-agents videosdk-plugins-silero videosdk-plugins-turn-detector videosdk-plugins-deepgram videosdk-plugins-openai videosdk-plugins-elevenlabs
2

1VIDEOSDK_API_KEY=your_videosdk_api_key
2OPENAI_API_KEY=your_openai_api_key
3DEEPGRAM_API_KEY=your_deepgram_api_key
4ELEVENLABS_API_KEY=your_elevenlabs_api_key
5

1import asyncio, os
2from videosdk.agents import Agent, AgentSession, CascadingPipeline, JobContext, RoomOptions, WorkerJob, ConversationFlow
3from videosdk.plugins.silero import SileroVAD
4from videosdk.plugins.turn_detector import TurnDetector, pre_download_model
5from videosdk.plugins.deepgram import DeepgramSTT
6from videosdk.plugins.openai import OpenAILLM
7from videosdk.plugins.elevenlabs import ElevenLabsTTS
8from typing import AsyncIterator
9
10# Pre-downloading the Turn Detector model
11pre_download_model()
12
13agent_instructions = "You are a professional and courteous AI Voice Agent designed specifically for a voice AI call center environment. Your persona is that of a knowledgeable, efficient, and empathetic virtual call center representative. 
14
15Capabilities:
16- Answer customer inquiries about products, services, account information, and general support topics relevant to the business.
17- Route calls or escalate issues to human agents when necessary, following predefined escalation protocols.
18- Collect and verify customer information securely and accurately.
19- Provide clear, concise, and accurate information based on the company's knowledge base and guidelines.
20- Handle multiple types of customer requests, including troubleshooting, order status, billing questions, and appointment scheduling.
21- Maintain a friendly, patient, and professional tone at all times.
22
23Constraints and Limitations:
24- Do not provide personal opinions, make promises on behalf of the company, or offer information outside the approved knowledge base.
25- Never collect sensitive information such as full credit card numbers, passwords, or social security numbers.
26- If a request is outside your scope or requires human intervention, politely inform the caller and offer to transfer or escalate the call.
27- Always adhere to privacy, security, and compliance guidelines as outlined by the company.
28- Clearly state that you are an AI voice agent at the beginning of each call.
29- Do not attempt to diagnose or resolve issues that require specialized human expertise; escalate such cases promptly.
30
31Your goal is to provide efficient, accurate, and friendly support, ensuring a seamless customer experience while respecting all privacy and security protocols."
32
33class MyVoiceAgent(Agent):
34    def __init__(self):
35        super().__init__(instructions=agent_instructions)
36    async def on_enter(self): await self.session.say("Hello! How can I help?")
37    async def on_exit(self): await self.session.say("Goodbye!")
38
39async def start_session(context: JobContext):
40    # Create agent and conversation flow
41    agent = MyVoiceAgent()
42    conversation_flow = ConversationFlow(agent)
43
44    # Create pipeline
45    pipeline = CascadingPipeline(
46        stt=DeepgramSTT(model="nova-2", language="en"),
47        llm=OpenAILLM(model="gpt-4o"),
48        tts=ElevenLabsTTS(model="eleven_flash_v2_5"),
49        vad=SileroVAD(threshold=0.35),
50        turn_detector=TurnDetector(threshold=0.8)
51    )
52
53    session = AgentSession(
54        agent=agent,
55        pipeline=pipeline,
56        conversation_flow=conversation_flow
57    )
58
59    try:
60        await context.connect()
61        await session.start()
62        # Keep the session running until manually terminated
63        await asyncio.Event().wait()
64    finally:
65        # Clean up resources when done
66        await session.close()
67        await context.shutdown()
68
69def make_context() -> JobContext:
70    room_options = RoomOptions(
71    #  room_id="YOUR_MEETING_ID",  # Set to join a pre-created room; omit to auto-create
72        name="VideoSDK Cascaded Agent",
73        playground=True
74    )
75
76    return JobContext(room_options=room_options)
77
78if __name__ == "__main__":
79    job = WorkerJob(entrypoint=start_session, jobctx=make_context)
80    job.start()
81

1curl -X POST \
2  -H "Authorization: YOUR_VIDEOSDK_API_KEY" \
3  -H "Content-Type: application/json" \
4  -d '{}' \
5  https://api.videosdk.live/v2/rooms
6

1agent_instructions = "You are a professional and courteous AI Voice Agent designed specifically for a voice AI call center environment. ..."
2
3class MyVoiceAgent(Agent):
4    def __init__(self):
5        super().__init__(instructions=agent_instructions)
6    async def on_enter(self):
7        await self.session.say("Hello! How can I help?")
8    async def on_exit(self):
9        await self.session.say("Goodbye!")
10

1pipeline = CascadingPipeline(
2    stt=DeepgramSTT(model="nova-2", language="en"),
3    llm=OpenAILLM(model="gpt-4o"),
4    tts=ElevenLabsTTS(model="eleven_flash_v2_5"),
5    vad=SileroVAD(threshold=0.35),
6    turn_detector=TurnDetector(threshold=0.8)
7)
8

1async def start_session(context: JobContext):
2    agent = MyVoiceAgent()
3    conversation_flow = ConversationFlow(agent)
4    pipeline = CascadingPipeline(
5        stt=DeepgramSTT(model="nova-2", language="en"),
6        llm=OpenAILLM(model="gpt-4o"),
7        tts=ElevenLabsTTS(model="eleven_flash_v2_5"),
8        vad=SileroVAD(threshold=0.35),
9        turn_detector=TurnDetector(threshold=0.8)
10    )
11    session = AgentSession(agent=agent, pipeline=pipeline, conversation_flow=conversation_flow)
12    try:
13        await context.connect()
14        await session.start()
15        await asyncio.Event().wait()
16    finally:
17        await session.close()
18        await context.shutdown()
19
20def make_context() -> JobContext:
21    room_options = RoomOptions(
22        # room_id="YOUR_MEETING_ID",
23        name="VideoSDK Cascaded Agent",
24        playground=True
25    )
26    return JobContext(room_options=room_options)
27
28if __name__ == "__main__":
29    job = WorkerJob(entrypoint=start_session, jobctx=make_context)
30    job.start()
31

1python main.py
2