Robust Speech Recognition in Noise

1sequenceDiagram
2    participant User
3    participant Agent
4    participant STT
5    participant LLM
6    participant TTS
7    User->>Agent: Speak
8    Agent->>STT: Convert Speech to Text
9    STT-->>Agent: Text
10    Agent->>LLM: Process Text
11    LLM-->>Agent: Response
12    Agent->>TTS: Convert Text to Speech
13    TTS-->>Agent: Audio
14    Agent->>User: Speak
15

1python -m venv venv
2source venv/bin/activate  # On Windows use `venv\\Scripts\\activate`
3

1pip install videosdk-agents videosdk-plugins-silero videosdk-plugins-turn-detector videosdk-plugins-deepgram videosdk-plugins-openai videosdk-plugins-elevenlabs
2

1import asyncio, os
2from videosdk.agents import Agent, AgentSession, CascadingPipeline, JobContext, RoomOptions, WorkerJob, ConversationFlow
3from videosdk.plugins.silero import SileroVAD
4from videosdk.plugins.turn_detector import TurnDetector, pre_download_model
5from videosdk.plugins.deepgram import DeepgramSTT
6from videosdk.plugins.openai import OpenAILLM
7from videosdk.plugins.elevenlabs import ElevenLabsTTS
8from typing import AsyncIterator
9
10# Pre-downloading the Turn Detector model
11pre_download_model()
12
13agent_instructions = "You are a 'Robust Speech Recognition Assistant' designed to operate effectively in noisy environments. Your primary role is to assist users by accurately recognizing and processing spoken commands or queries, even when there is significant background noise. You are particularly useful in environments such as busy offices, public transport, or crowded events.
14
15**Persona:** You are a friendly and efficient assistant, always ready to help users with their queries and tasks, regardless of the surrounding noise levels.
16
17**Capabilities:**
181. Accurately recognize and transcribe speech in noisy environments.
192. Answer general knowledge questions and provide information on a wide range of topics.
203. Assist with scheduling tasks, setting reminders, and managing to-do lists.
214. Provide navigation assistance and real-time updates on public transport schedules.
225. Offer recommendations for local services and amenities based on user preferences.
23
24**Constraints and Limitations:**
251. You are not capable of providing medical, legal, or financial advice and must always recommend consulting a professional for such queries.
262. You must include a disclaimer when providing information that may be subject to change, such as schedules or service availability.
273. You should not store or retain any personal user data beyond the session duration to ensure privacy and compliance with data protection regulations.
284. You may not always be able to filter out all background noise, and users should be advised to speak clearly and directly into the microphone for best results."
29
30class MyVoiceAgent(Agent):
31    def __init__(self):
32        super().__init__(instructions=agent_instructions)
33    async def on_enter(self): await self.session.say("Hello! How can I help?")
34    async def on_exit(self): await self.session.say("Goodbye!")
35
36async def start_session(context: JobContext):
37    # Create agent and conversation flow
38    agent = MyVoiceAgent()
39    conversation_flow = ConversationFlow(agent)
40
41    # Create pipeline
42    pipeline = CascadingPipeline(
43        stt=DeepgramSTT(model="nova-2", language="en"),
44        llm=OpenAILLM(model="gpt-4o"),
45        tts=ElevenLabsTTS(model="eleven_flash_v2_5"),
46        vad=SileroVAD(threshold=0.35),
47        turn_detector=TurnDetector(threshold=0.8)
48    )
49
50    session = AgentSession(
51        agent=agent,
52        pipeline=pipeline,
53        conversation_flow=conversation_flow
54    )
55
56    try:
57        await context.connect()
58        await session.start()
59        # Keep the session running until manually terminated
60        await asyncio.Event().wait()
61    finally:
62        # Clean up resources when done
63        await session.close()
64        await context.shutdown()
65
66def make_context() -> JobContext:
67    room_options = RoomOptions(
68    #  room_id="YOUR_MEETING_ID",  # Set to join a pre-created room; omit to auto-create
69        name="VideoSDK Cascaded Agent",
70        playground=True
71    )
72
73    return JobContext(room_options=room_options)
74
75if __name__ == "__main__":
76    job = WorkerJob(entrypoint=start_session, jobctx=make_context)
77    job.start()
78

1curl -X POST "https://api.videosdk.live/v1/meetings" -H "Authorization: YOUR_API_KEY" -H "Content-Type: application/json" -d '{}'
2

1python main.py
2