alex.a.alves/voiceagent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: integra VAD ao controle de turnos da conversa

Browse Source
This commit is contained in:
alex alves
2026-11-03 17:44:30 -03:00
parent d27c26c166
commit bdd2f6f67d
9 changed files with 770 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/voice_assistant/application/audio_history.py Normal file
View File

@@ -0,0 +1,32 @@
"""A bounded PCM history addressed by absolute input sample positions."""
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
class AudioHistory:
def __init__(self, history_ms: int) -> None:
if type(history_ms) is not int or history_ms <= 0:
raise ValueError("history_ms must be a positive integer.")
self._retained_samples = history_ms * CANONICAL_AUDIO_FORMAT.sample_rate // 1000
self._audio = bytearray()
self.start_sample = 0
self.end_sample = 0
def append(self, audio: bytes) -> None:
self._audio.extend(audio)
self.end_sample += len(audio) // CANONICAL_AUDIO_FORMAT.frame_size_bytes
def read(self, start: int, end: int) -> bytes:
if not self.start_sample <= start <= end <= self.end_sample:
raise ValueError("Requested audio is outside the retained input history.")
width = CANONICAL_AUDIO_FORMAT.frame_size_bytes
return bytes(self._audio[(start - self.start_sample) * width:(end - self.start_sample) * width])
def trim(self) -> None:
start = max(self.start_sample, self.end_sample - self._retained_samples)
del self._audio[:(start - self.start_sample) * CANONICAL_AUDIO_FORMAT.frame_size_bytes]
self.start_sample = start
def clear(self) -> None:
self._audio.clear()
self.start_sample = self.end_sample

14
src/voice_assistant/application/events.py
View File

@@ -41,6 +41,18 @@ class SpeechChunk:
audio_format: AudioFormat
@dataclass(frozen=True, slots=True)
class TurnOpened:
session_id: str
turn_id: int
@dataclass(frozen=True, slots=True)
class TurnInterrupted:
session_id: str
turn_id: int
@dataclass(frozen=True, slots=True)
class TurnCompleted:
session_id: str
@@ -58,5 +70,5 @@ class SessionFailure:
type VoiceEvent = (
TranscriptPartial | TranscriptFinal | AgentTextDelta | AgentReply
| SpeechChunk | TurnCompleted | SessionFailure
| SpeechChunk | TurnOpened | TurnInterrupted | TurnCompleted | SessionFailure
)

179
src/voice_assistant/application/voice_session.py
View File

@@ -6,7 +6,12 @@ from collections.abc import Awaitable, Callable
from contextlib import aclosing
from voice_assistant.agent.adapter import StreamingAgentAdapter
from voice_assistant.application.audio_history import AudioHistory
from voice_assistant.vad.adapter import VADAdapter
from voice_assistant.vad.models import SpeechStarted, SpeechEnded, VADEvent
from voice_assistant.application.events import (
TurnOpened,
TurnInterrupted,
AgentReply,
AgentTextDelta,
SessionFailure,
@@ -18,6 +23,7 @@ from voice_assistant.application.events import (
)
from voice_assistant.audio.models import AudioFormat, CANONICAL_AUDIO_FORMAT
from voice_assistant.stt.adapter import STTAdapter
from voice_assistant.turn_manager import TurnState
from voice_assistant.voice_pipeline import (
PipelineAgentResponse,
PipelineAudioChunk,
@@ -64,9 +70,30 @@ class VoiceSession:
emit: EventSink,
stt_stop_timeout: float = 10.0,
finalization_cancel_timeout: float = 5.0,
vad: VADAdapter | None = None,
pre_roll_ms: int = 200,
history_ms: int = 2000,
max_input_chunk_bytes: int = 65536,
) -> None:
if stt_stop_timeout <= 0 or finalization_cancel_timeout <= 0:
raise ValueError("Session timeouts must be positive.")
if type(pre_roll_ms) is not int or pre_roll_ms < 0:
raise ValueError("pre_roll_ms must be a nonnegative integer.")
if type(history_ms) is not int or history_ms <= 0 or pre_roll_ms > history_ms:
raise ValueError("history_ms must be positive and cover pre_roll_ms.")
if type(max_input_chunk_bytes) is not int or max_input_chunk_bytes <= 0:
raise ValueError("max_input_chunk_bytes must be a positive integer.")
self._vad = vad
self._input_lock = asyncio.Lock()
self._history = AudioHistory(history_ms)
self._pre_roll_samples = pre_roll_ms * CANONICAL_AUDIO_FORMAT.sample_rate // 1000
self._max_input_chunk_bytes = max_input_chunk_bytes
self._input_failed = False
self._input_finished = False
self._closing = False
self._capture_turn_id: int | None = None
self._sent_until = 0
self._last_voice_end = 0
self._pipeline = pipeline
self._create_stt = create_stt
self._emit = emit
@@ -116,7 +143,17 @@ class VoiceSession:
"SESSION_ALREADY_CLOSED", "This session is already closed."
)
def _require_manual_input(self) -> None:
if self._vad is not None:
raise VoiceSessionError(
"TURN_CONTROLLED_BY_VAD", "This session uses server-side turn detection.",
)
async def start_turn(self) -> int:
self._require_manual_input()
return await self._start_turn()
async def _start_turn(self) -> int:
self._require_open()
if self.active_turn_id is not None:
raise VoiceSessionError(
@@ -168,6 +205,12 @@ class VoiceSession:
async def send_audio(
self, turn_id: int, sequence_number: int, audio: bytes,
) -> None:
self._require_manual_input()
await self._send_audio(turn_id, sequence_number, audio)
async def _send_audio(
self, turn_id: int, sequence_number: int, audio: bytes,
) -> None:
self.validate_audio_input(turn_id)
CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
@@ -187,6 +230,10 @@ class VoiceSession:
) from exc
def stop_turn(self, turn_id: int) -> None:
self._require_manual_input()
self._stop_turn(turn_id)
def _stop_turn(self, turn_id: int) -> None:
self._require_open()
if self._stt is None:
raise RuntimeError("STT session has not been started.")
@@ -200,6 +247,12 @@ class VoiceSession:
)
async def interrupt(self, turn_id: int) -> None:
async with self._input_lock:
await self._interrupt(turn_id)
if self._capture_turn_id == turn_id:
self._capture_turn_id = None
async def _interrupt(self, turn_id: int) -> None:
self._require_open()
if not self._pipeline.is_active_turn(turn_id):
raise VoiceSessionError(
@@ -221,6 +274,11 @@ class VoiceSession:
) from exc
async def end(self) -> None:
self._closing = True
async with self._input_lock:
await self._end()
async def _end(self) -> None:
self._require_open()
try:
async with self._turn_transition_lock:
@@ -306,6 +364,12 @@ class VoiceSession:
or not self._pipeline.is_active_turn(turn_id)
):
return
turn = self._pipeline.turn_manager.active_turn
if self._vad is not None and turn is not None and turn.state == TurnState.LISTENING:
# A VAD trigger may produce no recognizable transcript.
self._pipeline.interrupt_if_active(turn_id)
await self._emit(TurnInterrupted(self.session_id, turn_id))
return
self._pipeline.complete_turn(turn_id)
self._completed_turns.add(turn_id)
await self._emit(TurnCompleted(self.session_id, turn_id))
@@ -389,6 +453,11 @@ class VoiceSession:
self._stt_events_task = None
async def aclose(self) -> None:
self._closing = True
async with self._input_lock:
await self._close_resources()
async def _close_resources(self) -> None:
"""Attempt every cleanup operation, including after partial startup.
The caller must shield this awaited cleanup from transport-scope
@@ -425,3 +494,113 @@ class VoiceSession:
"Failed to close %s provider. session_id=%s",
name, self.session_id,
)
if self._vad is not None:
try:
await self._vad.aclose()
except Exception:
logger.exception("Failed to close VAD. session_id=%s", self.session_id)
self._history.clear()
self._capture_turn_id = None
def _require_continuous_input(self) -> VADAdapter:
self._require_open()
if self._vad is None:
raise VoiceSessionError("INVALID_INPUT_MODE", "Continuous input requires VAD.")
if self._closing or self._input_finished:
raise VoiceSessionError("INPUT_CLOSED", "Continuous audio input is closed.")
if self._input_failed:
raise VoiceSessionError("VOICE_INPUT_FAILED", "Create a new session after input failure.")
return self._vad
async def receive_audio(self, audio: bytes) -> None:
"""Receive continuous PCM independently of client-managed turn IDs.
The channel serializes and validates its own packet sequence. This
operation validates PCM before advancing the application's timeline.
"""
CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
if len(audio) > self._max_input_chunk_bytes:
raise ValueError("Continuous audio chunk exceeds its configured size limit.")
async with self._input_lock:
vad = self._require_continuous_input()
self._history.append(audio)
try:
events = await vad.process(audio)
if self._closing:
raise VoiceSessionError("INPUT_CLOSED", "Session shutdown is in progress.")
await self._handle_voice_activity(events)
await self._forward_input_until(self._history.end_sample)
except asyncio.CancelledError:
self._input_failed = True
raise
except Exception as exc:
self._input_failed = True
if isinstance(exc, VoiceSessionError):
raise
raise VoiceSessionError(
"VOICE_INPUT_FAILED", str(exc), turn_id=self.active_turn_id,
) from exc
finally:
self._history.trim()
async def finish_audio_input(self) -> None:
"""Finish a normal input stream; session end/disconnect aborts instead."""
async with self._input_lock:
vad = self._require_continuous_input()
try:
events = await vad.flush()
if self._closing:
raise VoiceSessionError("INPUT_CLOSED", "Session shutdown is in progress.")
await self._handle_voice_activity(events)
self._input_finished = True
except asyncio.CancelledError:
self._input_failed = True
raise
except Exception as exc:
self._input_failed = True
if isinstance(exc, VoiceSessionError):
raise
raise VoiceSessionError(
"VOICE_INPUT_FAILED", str(exc), turn_id=self.active_turn_id,
) from exc
finally:
self._history.trim()
async def _handle_voice_activity(self, events: tuple[VADEvent, ...]) -> None:
for event in events:
if isinstance(event, SpeechStarted):
if self._capture_turn_id is not None:
raise ValueError("VAD emitted a duplicate speech start.")
if not self._history.start_sample <= event.sample_offset < self._history.end_sample:
raise ValueError("Speech start is outside retained audio; increase history_ms.")
previous = self.active_turn_id
if previous is not None:
await self._interrupt(previous)
await self._emit(TurnInterrupted(self.session_id, previous))
else:
await self._cancel_finalization_task()
turn_id = await self._start_turn()
self._capture_turn_id = turn_id
self._sent_until = max(
self._history.start_sample, self._last_voice_end,
event.sample_offset - self._pre_roll_samples,
)
await self._emit(TurnOpened(self.session_id, turn_id))
elif isinstance(event, SpeechEnded):
if not 0 <= event.sample_offset <= self._history.end_sample:
raise ValueError("Speech end is outside input audio.")
self._last_voice_end = event.sample_offset
if self._capture_turn_id is not None:
await self._forward_input_until(event.sample_offset)
turn_id = self._capture_turn_id
self._capture_turn_id = None
self._stop_turn(turn_id)
async def _forward_input_until(self, end_sample: int) -> None:
turn_id = self._capture_turn_id
if turn_id is None or end_sample <= self._sent_until:
return
audio = self._history.read(self._sent_until, end_sample)
await self._send_audio(turn_id, self.expected_audio_sequence, audio)
self._sent_until = end_sample

42
src/voice_assistant/audio_gateway/app/main.py
View File

@@ -13,7 +13,7 @@ from pydantic import BaseModel, ValidationError
from voice_assistant.application.events import (
AgentReply, AgentTextDelta, SessionFailure, SpeechChunk,
TranscriptFinal, TranscriptPartial, TurnCompleted, VoiceEvent,
TranscriptFinal, TranscriptPartial, TurnCompleted, TurnOpened, TurnInterrupted, VoiceEvent,
)
from voice_assistant.application.voice_session import VoiceSession, VoiceSessionError
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
@@ -26,9 +26,8 @@ from voice_assistant.audio_gateway.app.protocol import (
from voice_assistant.config.loader import load_config, load_provider_catalog
from voice_assistant.domain.session import SessionState
from voice_assistant.providers import (
create_agent_adapter, create_stt_adapter, create_tts_adapter,
create_agent_adapter, create_stt_adapter, create_tts_adapter, create_vad_adapter,
)
from voice_assistant.tts.adapter import TTSAdapter
from voice_assistant.turn_manager import TurnManager
from voice_assistant.voice_pipeline import VoicePipeline
@@ -81,6 +80,10 @@ async def send_voice_event(websocket: WebSocket, event: VoiceEvent) -> None:
payload = AgentDelta(session_id=sid, turn_id=tid, text=text)
case AgentReply(session_id=sid, turn_id=tid, text=text):
payload = AgentResponse(session_id=sid, turn_id=tid, text=text)
case TurnOpened(session_id=sid, turn_id=tid):
payload = TurnStarted(session_id=sid, turn_id=tid)
case TurnInterrupted(session_id=sid, turn_id=tid):
payload = Interrupted(session_id=sid, turn_id=tid)
case TurnCompleted(session_id=sid, turn_id=tid):
payload = TurnEnded(session_id=sid, turn_id=tid)
case SessionFailure():
@@ -93,20 +96,34 @@ async def send_voice_event(websocket: WebSocket, event: VoiceEvent) -> None:
await send_json(websocket, payload)
async def create_voice_session(websocket: WebSocket, session_id: str | None) -> VoiceSession:
"""Compose providers at the channel boundary, injecting application ports."""
async def create_voice_session(
websocket: WebSocket, session_id: str | None, *, server_vad: bool = False,
) -> VoiceSession:
"""Compose manual input or continuous input controlled by backend VAD."""
state = SessionState(session_id=session_id) if session_id is not None else SessionState()
agent = create_agent_adapter(config=app_config, catalog=provider_catalog)
tts: TTSAdapter | None = None
agent = None
tts = None
vad = None
try:
agent = create_agent_adapter(config=app_config, catalog=provider_catalog)
tts = create_tts_adapter(config=app_config, catalog=provider_catalog)
if server_vad:
vad = create_vad_adapter(config=app_config, catalog=provider_catalog)
pipeline = VoicePipeline(
session=state, turn_manager=create_turn_manager(), agent=agent, tts=tts,
)
return VoiceSession(
pipeline=pipeline,
create_stt=partial(create_stt_adapter, config=app_config, catalog=provider_catalog),
emit=partial(send_voice_event, websocket),
stt_stop_timeout=STT_STOP_TIMEOUT_SECONDS,
finalization_cancel_timeout=FINALIZATION_CANCEL_TIMEOUT_SECONDS,
vad=vad,
**app_config.voice_input,
)
except BaseException:
# A partially composed application has not yet acquired ownership.
with anyio.CancelScope(shield=True):
for name, provider in (("Agent", agent), ("TTS", tts)):
for name, provider in (("Agent", agent), ("TTS", tts), ("VAD", vad)):
if provider is None:
continue
try:
@@ -116,13 +133,6 @@ async def create_voice_session(websocket: WebSocket, session_id: str | None) ->
"Failed to close %s after session composition failure.", name,
)
raise
return VoiceSession(
pipeline=pipeline,
create_stt=partial(create_stt_adapter, config=app_config, catalog=provider_catalog),
emit=partial(send_voice_event, websocket),
stt_stop_timeout=STT_STOP_TIMEOUT_SECONDS,
finalization_cancel_timeout=FINALIZATION_CANCEL_TIMEOUT_SECONDS,
)
@app.websocket("/ws/audio")

1
src/voice_assistant/config/models.py
View File

@@ -72,3 +72,4 @@ class AppConfig(BaseModel):
tts: ComponentConfig
agent: ComponentConfig
vad: ComponentConfig | None = None
voice_input: dict[str, Any] = Field(default_factory=dict)