alex.a.alves/voiceagent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: establish voice assistant V1 baseline

Browse Source
This commit is contained in:
Alex Alves
2026-10-09 18:31:21 +00:00
commit cebcb3fd61
57 changed files with 9501 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/voice_assistant/__init__.py Normal file
View File

@@ -0,0 +1,2 @@
def main() -> None:
print("Hello from voice-assistant!")

7
src/voice_assistant/agent/__init__.py Normal file
View File

@@ -0,0 +1,7 @@
from .adapter import AgentAdapter
from .mock import MockAgentAdapter
__all__ = [
"AgentAdapter",
"MockAgentAdapter",
]

14
src/voice_assistant/agent/adapter.py Normal file
View File

@@ -0,0 +1,14 @@
from abc import ABC, abstractmethod
class AgentAdapter(ABC):
@abstractmethod
async def process(
self,
session_id: str,
turn_id: int,
text: str,
) -> str:
"""Process the final text of a turn and return the response."""
raise NotImplementedError

24
src/voice_assistant/agent/mock.py Normal file
View File

@@ -0,0 +1,24 @@
from .adapter import AgentAdapter
class MockAgentAdapter(AgentAdapter):
def __init__(self) -> None:
self.calls: list[
tuple[str, int, str]
] = []
async def process(
self,
session_id: str,
turn_id: int,
text: str,
) -> str:
self.calls.append(
(
session_id,
turn_id,
text,
)
)
return f"Resposta simulada para: {text}"

0
src/voice_assistant/audio/__init__.py Normal file
View File

63
src/voice_assistant/audio/models.py Normal file
View File

@@ -0,0 +1,63 @@
"""
Canonical audio models for the Voice Assistant.
Purpose:
Define the provider-independent audio format used internally by
the Voice Core.
Responsibilities:
- Describe an audio format in a typed structure.
- Define the canonical audio format used by the Voice Core.
- Provide a shared audio contract for transports and providers.
Non-responsibilities:
- Encoding or decoding audio.
- Resampling audio.
- Converting provider-specific audio formats.
- Managing WebSocket audio frames.
- Performing STT or TTS operations.
Architecture:
Client / Transport
|
v
Canonical Audio
PCM S16LE / 16 kHz / mono
|
v
Voice Core
|
+-----+-----+
| |
v v
STT TTS
Adapter Adapter
Provider-specific format conversion, when required, belongs
inside the corresponding provider adapter.
"""
from typing import Literal
from pydantic import BaseModel
class AudioFormat(BaseModel):
"""
Describe one PCM audio format.
This model is independent from WebSocket, STT, TTS, or any
concrete provider.
"""
encoding: Literal["pcm_s16le"]
sample_rate: int
channels: int
CANONICAL_AUDIO_FORMAT = AudioFormat(
encoding="pcm_s16le",
sample_rate=16000,
channels=1,
)

0
src/voice_assistant/audio_gateway/__init__.py Normal file
View File

0
src/voice_assistant/audio_gateway/app/__init__.py Normal file
View File

914
src/voice_assistant/audio_gateway/app/main.py Normal file
View File

@@ -0,0 +1,914 @@
"""
Audio Gateway application and WebSocket transport for the Voice Assistant.
Purpose:
Expose the realtime audio protocol and connect client sessions
to the VoicePipeline orchestration layer.
Responsibilities:
- Accept and validate WebSocket protocol messages.
- Receive canonical PCM audio frames.
- Create and configure session-scoped components.
- Forward STT events and TTS audio to connected clients.
- Delegate semantic turn lifecycle to VoicePipeline.
- Coordinate transport cleanup on disconnect.
- Report protocol and cleanup errors.
Non-responsibilities:
- Implementing STT, Agent, or TTS provider logic.
- Managing semantic turn state directly.
- Loading provider implementations dynamically.
- Implementing provider-specific cancellation.
Architecture:
WebSocket Client
|
v
Audio Gateway
|
v
VoicePipeline
| | |
STT Agent TTS
"""
import json
import logging
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError
import os
import asyncio
from voice_assistant.voice_pipeline import VoicePipeline
from voice_assistant.stt.adapter import STTAdapter
from voice_assistant.tts.adapter import TTSAdapter
from voice_assistant.turn_manager import (
TurnManager,
TurnState,
)
from voice_assistant.agent.adapter import AgentAdapter
from voice_assistant.audio.models import (
CANONICAL_AUDIO_FORMAT,
)
from voice_assistant.providers import (
create_agent_adapter,
create_stt_adapter,
create_tts_adapter,
)
from voice_assistant.config.loader import (
load_config,
load_provider_catalog,
)
from voice_assistant.audio_gateway.app.protocol import (
PROTOCOL_VERSION,
AUDIO_ENCODING,
AUDIO_SAMPLE_RATE,
AUDIO_CHANNELS,
AudioChunk,
AudioStart,
AudioStop,
Interrupt,
SessionEnd,
SessionStart,
error_event,
AgentResponse,
Interrupted,
SessionEnded,
SessionReady,
STTFinal,
STTPartial,
TurnEnded,
TurnStarted,
)
from voice_assistant.audio_gateway.app.session import SessionState
# logger configuration
logger = logging.getLogger(__name__)
app = FastAPI(
title="Voice Assistant Audio Gateway",
version=PROTOCOL_VERSION,
)
app_config = load_config()
provider_catalog = load_provider_catalog()
MAX_AUDIO_CHUNK_BYTES = 64 * 1024 # 64 KB
def create_turn_manager() -> TurnManager:
"""Create the turn manager used by the audio gateway."""
return TurnManager()
async def send_json(
websocket: WebSocket,
payload: BaseModel | dict,
) -> None:
"""
Serialize and send one protocol event through the WebSocket.
Typed Pydantic protocol models are preferred. Dictionary support
remains temporarily available for error events until ErrorEvent
is migrated to the typed protocol.
"""
if isinstance(payload, BaseModel):
data = payload.model_dump()
else:
data = payload
await websocket.send_text(
json.dumps(
data,
separators=(",", ":"),
)
)
async def forward_stt_events(
websocket: WebSocket,
stt: STTAdapter,
pipeline: VoicePipeline,
session_id: str,
turn_id: int,
) -> None:
async for event in stt.events():
if not pipeline.is_active_turn(turn_id):
logger.debug(
"Discarding stale STT event. session_id=%s turn_id=%s",
session_id,
turn_id,
)
break
if event.type == "partial":
pipeline.on_partial(
turn_id,
event.text,
)
await send_json(
websocket,
STTPartial(
session_id=session_id,
turn_id=turn_id,
text=event.text,
),
)
elif event.type == "final":
pipeline.on_final(
turn_id,
event.text,
)
await send_json(
websocket,
STTFinal(
session_id=session_id,
turn_id=turn_id,
text=event.text,
),
)
result = await pipeline.process_transcript(
session_id=session_id,
turn_id=turn_id,
text=event.text,
)
await send_json(
websocket,
AgentResponse(
session_id=session_id,
turn_id=turn_id,
text=result.response_text,
),
)
async for audio_chunk in pipeline.synthesize_response(
result.response_text,
):
await websocket.send_bytes(audio_chunk)
pipeline.on_tts_complete(turn_id)
@app.get("/health")
async def health() -> dict[str, str]:
return {"status": "ok"}
@app.get("/ready")
async def ready() -> dict[str, str]:
return {"status": "ready"}
@app.websocket("/ws/audio")
async def audio_socket(websocket: WebSocket) -> None:
await websocket.accept()
session: SessionState | None = None
pending_audio_chunk: AudioChunk | None = None
stt: STTAdapter | None = None
stt_events_task: asyncio.Task | None = None
turn_manager = create_turn_manager()
agent: AgentAdapter | None = None
tts: TTSAdapter | None = None
pipeline: VoicePipeline | None = None
try:
while True:
message = await websocket.receive()
if message.get("type") == "websocket.disconnect":
break
# ---------------------------------------------------------
# Binary WebSocket frame -> PCM
# ---------------------------------------------------------
if message.get("bytes") is not None:
if session is None:
await send_json(
websocket,
error_event(
"SESSION_NOT_FOUND",
"session.start is required before audio.",
),
)
continue
if pending_audio_chunk is None:
await send_json(
websocket,
error_event(
"INVALID_AUDIO_FRAME",
"Binary PCM frame must be preceded by audio.chunk.",
session_id=session.session_id,
turn_id=session.active_turn_id,
),
)
continue
audio_payload = message["bytes"]
if len(audio_payload) > MAX_AUDIO_CHUNK_BYTES:
await send_json(
websocket,
error_event(
"MESSAGE_TOO_LARGE",
(
"Audio chunk exceeds maximum size of "
f"{MAX_AUDIO_CHUNK_BYTES} bytes."
),
session_id=session.session_id,
turn_id=pending_audio_chunk.turn_id,
details={
"max_bytes": MAX_AUDIO_CHUNK_BYTES,
"received_bytes": len(audio_payload),
},
),
)
pending_audio_chunk = None
continue
try:
session.accept_audio_chunk(
pending_audio_chunk.turn_id,
pending_audio_chunk.sequence_number,
len(audio_payload),
)
except (RuntimeError, ValueError) as exc:
await send_json(
websocket,
error_event(
"AUDIO_SEQUENCE_ERROR",
str(exc),
session_id=session.session_id,
turn_id=pending_audio_chunk.turn_id,
details={
"expected_sequence_number":
session.expected_audio_sequence,
},
),
)
pending_audio_chunk = None
continue
# echo do PCM para validar o transporte.
if stt is None:
await send_json(
websocket,
error_event(
"STT_NOT_STARTED",
"audio.start is required before audio.",
session_id=session.session_id,
turn_id=session.active_turn_id,
),
)
pending_audio_chunk = None
continue
try:
await stt.send_audio(audio_payload)
except (RuntimeError, ValueError) as exc:
await send_json(
websocket,
error_event(
"STT_AUDIO_ERROR",
str(exc),
session_id=session.session_id,
turn_id=pending_audio_chunk.turn_id,
retryable=True,
),
)
pending_audio_chunk = None
continue
pending_audio_chunk = None
continue
# ---------------------------------------------------------
# Text WebSocket frame -> JSON control event
# ---------------------------------------------------------
text = message.get("text")
if text is None:
await send_json(
websocket,
error_event(
"INVALID_MESSAGE",
"Unsupported WebSocket frame.",
),
)
continue
try:
payload = json.loads(text)
except json.JSONDecodeError:
await send_json(
websocket,
error_event(
"INVALID_MESSAGE",
"Message must contain valid JSON.",
),
)
continue
event_type = payload.get("type")
if not isinstance(event_type, str):
await send_json(
websocket,
error_event(
"INVALID_MESSAGE",
"Field 'type' is required.",
),
)
continue
# ---------------------------------------------------------
# session.start
# ---------------------------------------------------------
if event_type == "session.start":
if session is not None:
await send_json(
websocket,
error_event(
"SESSION_ALREADY_STARTED",
"This WebSocket already has an active session.",
session_id=session.session_id,
),
)
continue
try:
request = SessionStart.model_validate(payload)
except ValidationError as exc:
await send_json(
websocket,
error_event(
"INVALID_MESSAGE",
"Invalid session.start payload.",
details={"errors": exc.errors()},
),
)
continue
audio = request.audio
if (
audio.encoding != AUDIO_ENCODING
or audio.sample_rate != AUDIO_SAMPLE_RATE
or audio.channels != AUDIO_CHANNELS
):
await send_json(
websocket,
error_event(
"INVALID_AUDIO_FORMAT",
(
"Only PCM S16LE, mono, 16 kHz "
"is supported in the MVP."
),
details={
"expected": {
"encoding": AUDIO_ENCODING,
"sample_rate": AUDIO_SAMPLE_RATE,
"channels": AUDIO_CHANNELS,
}
},
),
)
continue
session = SessionState(
session_id=request.session_id or SessionState().session_id
)
agent = create_agent_adapter(
config=app_config,
catalog=provider_catalog,
)
tts = create_tts_adapter(
config=app_config,
catalog=provider_catalog,
)
pipeline = VoicePipeline(
session=session,
turn_manager=turn_manager,
agent=agent,
tts=tts,
)
await send_json(
websocket,
SessionReady(
protocol_version=PROTOCOL_VERSION,
session_id=session.session_id,
audio=CANONICAL_AUDIO_FORMAT,
),
)
continue
# ---------------------------------------------------------
# Todos os eventos abaixo exigem sessão ativa
# ---------------------------------------------------------
if session.closed:
await send_json(
websocket,
error_event(
"SESSION_ALREADY_CLOSED",
"This session is already closed.",
session_id=session.session_id,
),
)
continue
# ---------------------------------------------------------
# audio.start
# ---------------------------------------------------------
if event_type == "audio.start":
turn_id: int | None = None
try:
if pipeline is None:
raise RuntimeError(
"Voice pipeline is not initialized."
)
turn = pipeline.start_turn()
turn_id = turn.turn_id
# Each turn receives a fresh STT provider instance.
stt = create_stt_adapter(
config=app_config,
catalog=provider_catalog,
)
await stt.start_session()
stt_events_task = asyncio.create_task(
forward_stt_events(
websocket=websocket,
stt=stt,
pipeline=pipeline,
session_id=session.session_id,
turn_id=turn_id,
)
)
pipeline.register_turn_task(stt_events_task)
except (ValidationError, RuntimeError, ValueError) as exc:
# Preserve the original startup failure.
startup_error = exc
cleanup_complete = True
# Cancel and await any task created during startup.
if stt_events_task is not None:
task = stt_events_task
try:
if (
pipeline is not None
and pipeline.turn_task is task
):
await pipeline.cancel_turn_task()
else:
if not task.done():
task.cancel()
try:
await asyncio.wait_for(
asyncio.shield(task),
timeout=5.0,
)
except asyncio.CancelledError:
if not task.cancelled():
raise
except Exception:
cleanup_complete = False
logger.exception(
"Failed to cancel task during "
"audio.start rollback."
)
if task.done():
stt_events_task = None
else:
cleanup_complete = False
# Abort the partially initialized STT provider.
if stt is not None:
try:
await stt.abort_session()
except Exception:
cleanup_complete = False
logger.exception(
"Failed to abort STT during "
"audio.start rollback."
)
# Only release semantic state after execution
# has stopped and provider cleanup succeeded.
if cleanup_complete:
if turn_id is not None and pipeline is not None:
pipeline.interrupt_turn(turn_id)
stt = None
pending_audio_chunk = None
await send_json(
websocket,
error_event(
"STT_START_FAILED",
str(startup_error),
session_id=session.session_id,
turn_id=turn_id,
retryable=cleanup_complete,
),
)
continue
# Startup completed successfully.
await send_json(
websocket,
TurnStarted(
session_id=session.session_id,
turn_id=turn_id,
),
)
continue
# ---------------------------------------------------------
# audio.chunk
# ---------------------------------------------------------
if event_type == "audio.chunk":
try:
request = AudioChunk.model_validate(payload)
if session.active_turn_id != request.turn_id:
raise ValueError(
f"Unexpected turn_id={request.turn_id}; "
f"active_turn_id={session.active_turn_id}"
)
if pending_audio_chunk is not None:
raise ValueError(
"Previous audio chunk is still pending PCM data."
)
pending_audio_chunk = request
except (ValidationError, ValueError) as exc:
await send_json(
websocket,
error_event(
"AUDIO_SEQUENCE_ERROR",
str(exc),
session_id=session.session_id,
turn_id=payload.get("turn_id"),
details={
"expected_sequence_number":
session.expected_audio_sequence,
},
),
)
continue
continue
# ---------------------------------------------------------
# audio.stop
# ---------------------------------------------------------
if event_type == "audio.stop":
try:
request = AudioStop.model_validate(payload)
if stt is None:
raise RuntimeError(
"STT session has not been started."
)
if pending_audio_chunk is not None:
raise ValueError(
"audio.stop received while an audio chunk is pending."
)
if agent is None:
raise RuntimeError(
"Agent adapter is not initialized."
)
if tts is None:
raise RuntimeError(
"TTS adapter is not initialized."
)
if pipeline is None:
raise RuntimeError(
"Voice pipeline is not initialized."
)
# Finish the STT session normally.
await stt.stop_session()
await pipeline.wait_turn_task()
stt_events_task = None
try:
pipeline.complete_turn(request.turn_id)
except (RuntimeError, ValueError) as exc:
raise RuntimeError(
"Turn pipeline did not complete successfully."
) from exc
except (ValidationError, RuntimeError, ValueError) as exc:
await send_json(
websocket,
error_event(
"INVALID_TURN",
str(exc),
session_id=session.session_id,
turn_id=payload.get("turn_id"),
),
)
continue
stt = None
pending_audio_chunk = None
await send_json(
websocket,
TurnEnded(
session_id=session.session_id,
turn_id=request.turn_id,
),
)
continue
# ---------------------------------------------------------
# interrupt
# ---------------------------------------------------------
if event_type == "interrupt":
try:
request = Interrupt.model_validate(payload)
except ValidationError as exc:
await send_json(
websocket,
error_event(
"INVALID_MESSAGE",
"Invalid interrupt payload.",
session_id=session.session_id,
details={"errors": exc.errors()},
),
)
continue
if pipeline is None:
await send_json(
websocket,
error_event(
"INVALID_TURN",
"Voice pipeline is not initialized.",
session_id=session.session_id,
turn_id=request.turn_id,
),
)
continue
# Only cancel resources belonging to the requested active turn.
if pipeline.is_active_turn(request.turn_id):
try:
# Cancel and await STT/Agent/TTS execution before
# confirming the interruption to the client.
await pipeline.cancel_turn_task()
# Abort transcription without requesting a final result.
if stt is not None:
await stt.abort_session()
# Release semantic turn state after execution stops.
pipeline.interrupt_if_active(request.turn_id)
# Clear references to the interrupted execution.
stt_events_task = None
pending_audio_chunk = None
except Exception as exc:
await send_json(
websocket,
error_event(
"TURN_CANCEL_FAILED",
str(exc),
session_id=session.session_id,
turn_id=request.turn_id,
),
)
continue
else:
# Preserve the existing tolerant behavior for unknown turns.
pipeline.interrupt_if_active(request.turn_id)
await send_json(
websocket,
Interrupted(
session_id=session.session_id,
turn_id=request.turn_id,
),
)
continue
# ---------------------------------------------------------
# session.end
# ---------------------------------------------------------
if event_type == "session.end":
try:
SessionEnd.model_validate(payload)
if pipeline is None:
raise RuntimeError(
"Voice pipeline is not initialized."
)
# Cancel and await any running STT/Agent/TTS task.
await pipeline.cancel_turn_task()
# Abort STT without waiting for a final transcript.
if stt is not None:
await stt.abort_session()
# Close semantic turn and session state.
pipeline.close_session()
# Release Gateway-owned references.
stt_events_task = None
stt = None
pending_audio_chunk = None
except (ValidationError, RuntimeError, ValueError) as exc:
await send_json(
websocket,
error_event(
"SESSION_END_FAILED",
str(exc),
session_id=session.session_id,
turn_id=session.active_turn_id,
retryable=True,
),
)
continue
await send_json(
websocket,
SessionEnded(
session_id=session.session_id,
),
)
break
# ---------------------------------------------------------
# Evento desconhecido
# ---------------------------------------------------------
await send_json(
websocket,
error_event(
"UNSUPPORTED_EVENT",
f"Unsupported event type: {event_type!r}.",
session_id=session.session_id,
turn_id=session.active_turn_id,
),
)
except WebSocketDisconnect:
# The finally block owns connection cleanup.
pass
finally:
# Cancel and await the task owned by VoicePipeline.
if pipeline is not None:
try:
await pipeline.cancel_turn_task()
except Exception:
logger.exception(
"Failed to cancel voice turn task. session_id=%s",
session.session_id if session else None,
)
# A task may have been created before registration failed.
# Handle it independently if the pipeline does not own it.
if stt_events_task is not None:
task = stt_events_task
if (
pipeline is None
or pipeline.turn_task is not task
):
try:
if not task.done():
task.cancel()
try:
await asyncio.wait_for(
asyncio.shield(task),
timeout=5.0,
)
except asyncio.CancelledError:
if not task.cancelled():
raise
except TimeoutError:
logger.error(
"Unregistered STT task cleanup timed out. "
"session_id=%s",
session.session_id if session else None,
)
except Exception:
logger.exception(
"Failed to clean up unregistered STT task. "
"session_id=%s",
session.session_id if session else None,
)
if task.done():
stt_events_task = None
# Abort STT without waiting for a final transcript.
if stt is not None:
try:
await stt.abort_session()
except Exception:
logger.exception(
"Failed to abort STT session. session_id=%s",
session.session_id if session else None,
)
# Release session and semantic turn state.
try:
if pipeline is not None:
pipeline.close_session()
elif session is not None:
session.close()
except Exception:
logger.exception(
"Failed to close voice session. session_id=%s",
session.session_id if session else None,
)

163
src/voice_assistant/audio_gateway/app/protocol.py Normal file
View File

@@ -0,0 +1,163 @@
from __future__ import annotations
from typing import Any, Literal
from pydantic import BaseModel, Field
from voice_assistant.audio.models import (
AudioFormat,
CANONICAL_AUDIO_FORMAT,
)
PROTOCOL_VERSION = "1.0"
AUDIO_ENCODING = "pcm_s16le"
AUDIO_SAMPLE_RATE = 16000
AUDIO_CHANNELS = 1
class SessionStart(BaseModel):
"""Request that initializes a new Voice Assistant session."""
type: Literal["session.start"] = "session.start"
session_id: str | None = None
audio: AudioFormat = Field(
default_factory=lambda: CANONICAL_AUDIO_FORMAT.model_copy()
)
class ErrorEvent(BaseModel):
"""Error response sent by the Audio Gateway."""
type: Literal["error"] = "error"
session_id: str | None = None
turn_id: int | None = None
code: str
message: str
retryable: bool = False
details: dict[str, Any] = Field(
default_factory=dict,
)
class AudioStart(BaseModel):
"""Request that starts a new audio turn."""
type: Literal["audio.start"] = "audio.start"
class AudioChunk(BaseModel):
"""Metadata describing the next binary audio chunk."""
type: Literal["audio.chunk"] = "audio.chunk"
turn_id: int
sequence_number: int
class AudioStop(BaseModel):
"""Request that finishes audio input for the active turn."""
type: Literal["audio.stop"] = "audio.stop"
turn_id: int
class Interrupt(BaseModel):
"""Request that interrupts the active turn."""
type: Literal["interrupt"] = "interrupt"
turn_id: int
class SessionEnd(BaseModel):
"""Request that closes the active session."""
type: Literal["session.end"] = "session.end"
class SessionReady(BaseModel):
"""Response confirming that a session is ready."""
type: Literal["session.ready"] = "session.ready"
protocol_version: str
session_id: str
audio: AudioFormat
class TurnStarted(BaseModel):
"""Response confirming that a new turn has started."""
type: Literal["turn.started"] = "turn.started"
session_id: str
turn_id: int
class STTPartial(BaseModel):
"""Partial transcription produced during a turn."""
type: Literal["stt.partial"] = "stt.partial"
session_id: str
turn_id: int
text: str
class STTFinal(BaseModel):
"""Final transcription produced for a turn."""
type: Literal["stt.final"] = "stt.final"
session_id: str
turn_id: int
text: str
class AgentResponse(BaseModel):
"""Text response produced by the configured Agent."""
type: Literal["agent.response"] = "agent.response"
session_id: str
turn_id: int
text: str
class TurnEnded(BaseModel):
"""Response confirming successful completion of a turn."""
type: Literal["turn.ended"] = "turn.ended"
session_id: str
turn_id: int
class Interrupted(BaseModel):
"""Response confirming interruption of a turn."""
type: Literal["interrupted"] = "interrupted"
session_id: str
turn_id: int
class SessionEnded(BaseModel):
"""Response confirming that a session has ended."""
type: Literal["session.ended"] = "session.ended"
session_id: str
def error_event(
code: str,
message: str,
*,
session_id: str | None = None,
turn_id: int | None = None,
retryable: bool = False,
details: dict[str, Any] | None = None,
) -> ErrorEvent:
"""Build a typed error protocol event."""
return ErrorEvent(
session_id=session_id,
turn_id=turn_id,
code=code,
message=message,
retryable=retryable,
details=details or {},
)

70
src/voice_assistant/audio_gateway/app/session.py Normal file
View File

@@ -0,0 +1,70 @@
from dataclasses import dataclass, field
from uuid import uuid4
@dataclass
class SessionState:
session_id: str = field(default_factory=lambda: str(uuid4()))
active_turn_id: int | None = None
next_turn_id: int = 1
expected_audio_sequence: int = 0
closed: bool = False
def start_turn(self) -> int:
if self.closed:
raise RuntimeError("Session is closed.")
if self.active_turn_id is not None:
raise RuntimeError("A turn is already active.")
self.active_turn_id = self.next_turn_id
self.next_turn_id += 1
self.expected_audio_sequence = 0
return self.active_turn_id
def accept_audio_chunk(
self,
turn_id: int,
sequence_number: int,
payload_size: int,
) -> None:
if self.closed:
raise RuntimeError("Session is closed.")
if self.active_turn_id != turn_id:
raise ValueError(
f"Unexpected turn_id={turn_id}; "
f"active_turn_id={self.active_turn_id}."
)
if sequence_number != self.expected_audio_sequence:
raise ValueError(
f"Expected sequence_number="
f"{self.expected_audio_sequence}, "
f"received={sequence_number}."
)
if payload_size == 0:
raise ValueError("Audio chunk cannot be empty.")
self.expected_audio_sequence += 1
def end_turn(self, turn_id: int) -> None:
if self.active_turn_id != turn_id:
raise ValueError(
f"Cannot end turn {turn_id}; "
f"active_turn_id={self.active_turn_id}."
)
self.active_turn_id = None
self.expected_audio_sequence = 0
def interrupt(self, turn_id: int) -> None:
if self.active_turn_id == turn_id:
self.active_turn_id = None
self.expected_audio_sequence = 0
def close(self) -> None:
self.active_turn_id = None
self.closed = True

0
src/voice_assistant/config/__init__.py Normal file
View File

109
src/voice_assistant/config/loader.py Normal file
View File

@@ -0,0 +1,109 @@
"""
Configuration loading for the Voice Assistant.
Purpose:
Load environment-specific YAML configuration and the provider
catalog, then validate both using the typed configuration models.
Responsibilities:
- Select the active environment.
- Load environment configuration.
- Load provider catalog metadata.
- Validate configuration through Pydantic models.
Non-responsibilities:
- Instantiating providers.
- Executing STT, Agent, or TTS functionality.
- Managing application lifecycle.
"""
import os
from pathlib import Path
from typing import Any
import yaml
from .models import AppConfig, ProviderCatalog
DEFAULT_CONFIG_DIR = Path("config")
def get_environment() -> str:
"""
Return the configured application environment.
Defaults to development for local execution.
"""
return os.getenv(
"APP_ENV",
"development",
)
def load_config(
environment: str | None = None,
config_dir: Path = DEFAULT_CONFIG_DIR,
) -> AppConfig:
"""Load and validate the selected environment configuration."""
environment = environment or get_environment()
config_path = config_dir / f"{environment}.yaml"
if not config_path.exists():
raise FileNotFoundError(
f"Configuration file not found: {config_path}"
)
with config_path.open(
"r",
encoding="utf-8",
) as file:
data: Any = yaml.safe_load(file)
if data is None:
raise ValueError(
f"Configuration file is empty: {config_path}"
)
if not isinstance(data, dict):
raise ValueError(
f"Configuration root must be a mapping: "
f"{config_path}"
)
return AppConfig.model_validate(data)
def load_provider_catalog(
config_dir: Path = DEFAULT_CONFIG_DIR,
) -> ProviderCatalog:
"""Load and validate the provider catalog."""
config_path = config_dir / "providers.yaml"
if not config_path.exists():
raise FileNotFoundError(
f"Provider catalog not found: {config_path}"
)
with config_path.open(
"r",
encoding="utf-8",
) as file:
data: Any = yaml.safe_load(file)
if data is None:
raise ValueError(
f"Provider catalog is empty: {config_path}"
)
if not isinstance(data, dict):
raise ValueError(
f"Provider catalog root must be a mapping: "
f"{config_path}"
)
return ProviderCatalog.model_validate(data)

73
src/voice_assistant/config/models.py Normal file
View File

@@ -0,0 +1,73 @@
"""
Typed configuration models for the Voice Assistant.
Purpose:
Define and validate the configuration structures used by the
application and by the dynamic provider loading system.
Responsibilities:
- Define the application configuration for STT, Agent, and TTS.
- Define the catalog structure for available providers.
- Represent provider-specific options without coupling the core
configuration model to a specific vendor or implementation.
- Describe provider capabilities for discovery and future tooling.
Non-responsibilities:
- Loading YAML files.
- Reading environment variables or secrets.
- Selecting or instantiating provider implementations.
- Executing STT, Agent, or TTS functionality.
Architecture:
YAML
|
v
Config Loader
|
v
AppConfig / ProviderCatalog
|
v
Provider Loader
|
v
STTAdapter / AgentAdapter / TTSAdapter
"""
from typing import Any
from pydantic import BaseModel, Field
class ProviderDefinition(BaseModel):
"""Metadata describing one available provider."""
name: str
implementation: str
capabilities: dict[str, bool] = Field(
default_factory=dict,
)
class ProviderCatalog(BaseModel):
"""Available providers grouped by component type."""
providers: dict[
str,
dict[str, ProviderDefinition],
] = Field(default_factory=dict)
class ComponentConfig(BaseModel):
"""Selected provider and its provider-specific runtime options."""
provider: str
options: dict[str, Any] = Field(default_factory=dict)
class AppConfig(BaseModel):
"""Provider selections for one application environment."""
stt: ComponentConfig
tts: ComponentConfig
agent: ComponentConfig

187
src/voice_assistant/providers.py Normal file
View File

@@ -0,0 +1,187 @@
"""
Dynamic provider loading and composition for the Voice Assistant.
Purpose:
Resolve provider names from the application configuration and
instantiate implementations that satisfy the corresponding adapter
contract.
Responsibilities:
- Resolve provider implementations from ProviderCatalog metadata.
- Import provider classes dynamically.
- Validate that the loaded implementation matches the expected adapter.
- Instantiate STT, Agent, and TTS providers using provider-specific options.
Non-responsibilities:
- Loading YAML files.
- Reading secrets directly.
- Managing WebSocket connections.
- Managing turn state.
- Executing provider functionality.
Architecture:
AppConfig + ProviderCatalog
|
v
Provider Loader
|
v
STTAdapter / AgentAdapter / TTSAdapter
|
v
concrete provider
"""
from importlib import import_module
from typing import Any
from voice_assistant.agent.adapter import AgentAdapter
from voice_assistant.config.models import (
AppConfig,
ComponentConfig,
ProviderCatalog,
)
from voice_assistant.stt.adapter import STTAdapter
from voice_assistant.tts.adapter import TTSAdapter
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
PROVIDER_CONTRACTS: dict[str, type] = {
"stt": STTAdapter,
"agent": AgentAdapter,
"tts": TTSAdapter,
}
def load_provider_class(
catalog: ProviderCatalog,
component: str,
provider: str,
) -> type:
"""
Load and validate a provider class from the provider catalog.
The catalog stores the import path, while this function ensures that
the resolved class implements the adapter contract associated with the
requested component.
"""
contract = PROVIDER_CONTRACTS.get(component)
if contract is None:
raise ValueError(
f"Unsupported provider component: {component!r}."
)
component_providers = catalog.providers.get(component)
if component_providers is None:
raise ValueError(
f"No providers are registered for component {component!r}."
)
definition = component_providers.get(provider)
if definition is None:
raise ValueError(
f"Provider {provider!r} is not registered for "
f"component {component!r}."
)
module_name, class_name = definition.implementation.split(
":",
maxsplit=1,
)
try:
module = import_module(module_name)
except ImportError as exc:
raise RuntimeError(
f"Could not import provider module "
f"{module_name!r} for {component!r}/{provider!r}."
) from exc
try:
provider_class = getattr(module, class_name)
except AttributeError as exc:
raise RuntimeError(
f"Provider class {class_name!r} was not found in "
f"module {module_name!r}."
) from exc
if not isinstance(provider_class, type):
raise TypeError(
f"Provider {component!r}/{provider!r} does not resolve "
f"to a class."
)
if not issubclass(provider_class, contract):
raise TypeError(
f"Provider {component!r}/{provider!r} must implement "
f"{contract.__name__}."
)
return provider_class
def _create_provider(
catalog: ProviderCatalog,
component: str,
config: ComponentConfig,
) -> Any:
"""
Instantiate the provider selected by a component configuration.
Provider-specific constructor arguments are kept inside `options`
so the common configuration model does not need to know the API of
every vendor.
"""
provider_class = load_provider_class(
catalog=catalog,
component=component,
provider=config.provider,
)
try:
return provider_class(**config.options)
except TypeError as exc:
raise TypeError(
f"Invalid options for provider "
f"{component!r}/{config.provider!r}: {exc}"
) from exc
def create_stt_adapter(
config: AppConfig,
catalog: ProviderCatalog,
) -> STTAdapter:
"""Create the configured STT adapter."""
return _create_provider(
catalog=catalog,
component="stt",
config=config.stt,
)
def create_agent_adapter(
config: AppConfig,
catalog: ProviderCatalog,
) -> AgentAdapter:
"""Create the configured Agent adapter."""
return _create_provider(
catalog=catalog,
component="agent",
config=config.agent,
)
def create_tts_adapter(
config: AppConfig,
catalog: ProviderCatalog,
) -> TTSAdapter:
"""Create the configured TTS adapter."""
return _create_provider(
catalog=catalog,
component="tts",
config=config.tts,
)

0
src/voice_assistant/stt/__init__.py Normal file
View File

85
src/voice_assistant/stt/adapter.py Normal file
View File

@@ -0,0 +1,85 @@
"""
Provider-independent Speech-to-Text adapter contract.
Purpose:
Define the interface that every STT provider must implement
to participate in the Voice Assistant pipeline.
Responsibilities:
- Start transcription sessions.
- Receive canonical PCM audio chunks.
- Finish transcription sessions normally.
- Abort transcription sessions during interruption or cleanup.
- Expose provider-independent transcript events.
Non-responsibilities:
- Managing Audio Gateway WebSocket connections.
- Managing SessionState or TurnManager.
- Executing Agent or TTS operations.
- Implementing provider-specific communication.
Architecture:
Audio Gateway / VoicePipeline
|
v
STTAdapter
|
+-------+-------+
| |
v v
NvidiaSTTAdapter MockSTTAdapter
Concrete providers implement this contract without exposing
their internal protocols to the Voice Core.
"""
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from .models import TranscriptEvent
class STTAdapter(ABC):
"""Abstract contract for Speech-to-Text providers."""
@abstractmethod
async def start_session(self) -> None:
"""Initialize a transcription session."""
raise NotImplementedError
@abstractmethod
async def send_audio(self, audio: bytes) -> None:
"""Send one canonical PCM audio chunk for transcription."""
raise NotImplementedError
@abstractmethod
async def stop_session(self) -> None:
"""
Finish transcription normally.
The provider may wait for its final transcription result
before releasing the session.
"""
raise NotImplementedError
@abstractmethod
async def abort_session(self) -> None:
"""
Abort transcription without waiting for a final result.
Implementations must:
- Stop accepting new audio.
- Release active provider resources.
- Unblock consumers waiting for transcript events.
- Avoid waiting for final transcription.
- Support repeated calls safely.
The provider must not emit new transcription results after
the abort operation has completed.
"""
raise NotImplementedError
@abstractmethod
def events(self) -> AsyncIterator[TranscriptEvent]:
"""Expose the asynchronous stream of transcript events."""
raise NotImplementedError

161
src/voice_assistant/stt/mock.py Normal file
View File

@@ -0,0 +1,161 @@
"""
In-memory mock STT provider for the Voice Assistant.
Purpose:
Simulate a realtime STT provider without external services.
Responsibilities:
- Simulate transcription session lifecycle.
- Validate incoming audio chunks.
- Expose partial and final transcript events.
- Support normal stop and immediate abort.
- Provide deterministic behavior for automated tests.
Non-responsibilities:
- Performing real speech recognition.
- Managing network connections.
- Managing voice turns or Gateway sessions.
- Executing Agent or TTS operations.
Architecture:
Tests / VoicePipeline
|
v
STTAdapter
|
v
MockSTTAdapter
|
v
Async event queue
"""
import asyncio
from collections.abc import AsyncIterator
from .adapter import STTAdapter
from .models import TranscriptEvent
class MockSTTAdapter(STTAdapter):
def __init__(self) -> None:
self._events: asyncio.Queue[
TranscriptEvent | None
] = asyncio.Queue()
self._started = False
async def start_session(self) -> None:
self._started = True
self._events = asyncio.Queue()
async def send_audio(self, audio: bytes) -> None:
if not self._started:
raise RuntimeError(
"STT session has not been started."
)
if not audio:
raise ValueError(
"Audio chunk cannot be empty."
)
async def stop_session(self) -> None:
if not self._started:
return
self._started = False
await self._events.put(None)
async def abort_session(self) -> None:
"""
Abort transcription and discard pending transcript events.
Unlike stop_session(), abort_session() must not deliver
queued partial or final transcription results.
"""
if not self._started:
return
# Reject further audio and transcript events.
self._started = False
# Discard queued transcript events from the aborted session.
while True:
try:
self._events.get_nowait()
except asyncio.QueueEmpty:
break
# Unblock consumers waiting for the next event.
await self._events.put(None)
def events(self) -> AsyncIterator[TranscriptEvent]:
return self._event_stream()
async def _event_stream(
self,
) -> AsyncIterator[TranscriptEvent]:
while True:
event = await self._events.get()
if event is None:
break
yield event
async def emit_partial(self, text: str) -> None:
if not self._started:
raise RuntimeError(
"STT session has not been started."
)
await self._events.put(
TranscriptEvent(
type="partial",
text=text,
)
)
async def emit_final(self, text: str) -> None:
if not self._started:
raise RuntimeError(
"STT session has not been started."
)
await self._events.put(
TranscriptEvent(
type="final",
text=text,
)
)
def emit_partial_now(self, text: str) -> None:
if not self._started:
raise RuntimeError(
"STT session has not been started."
)
self._events.put_nowait(
TranscriptEvent(
type="partial",
text=text,
)
)
def emit_final_now(self, text: str) -> None:
if not self._started:
raise RuntimeError(
"STT session has not been started."
)
self._events.put_nowait(
TranscriptEvent(
type="final",
text=text,
)
)

11
src/voice_assistant/stt/models.py Normal file
View File

@@ -0,0 +1,11 @@
from dataclasses import dataclass
from typing import Literal
TranscriptType = Literal["partial", "final"]
@dataclass(slots=True)
class TranscriptEvent:
type: TranscriptType
text: str

430
src/voice_assistant/stt/nvidia.py Normal file
View File

@@ -0,0 +1,430 @@
"""
NVIDIA realtime STT provider for the Voice Assistant.
Purpose:
Implement the generic STTAdapter contract using NVIDIA Speech NIM.
Responsibilities:
- Create and manage NVIDIA realtime transcription sessions.
- Send canonical PCM audio to the NVIDIA ASR service.
- Receive provider-specific transcription events.
- Translate NVIDIA events into provider-independent TranscriptEvent objects.
- Manage provider connection lifecycle and cleanup.
Non-responsibilities:
- Managing WebSocket client connections to the Audio Gateway.
- Managing Voice Assistant sessions or turns.
- Executing Agent or TTS functionality.
- Defining the canonical Voice Core audio format.
Architecture:
Voice Core
|
| canonical PCM audio
v
STTAdapter
|
v
NvidiaSTTAdapter
|
v
NVIDIA Speech NIM
NVIDIA-specific HTTP, WebSocket, session, and event details remain
isolated inside this adapter.
"""
from __future__ import annotations
import asyncio
import base64
import json
from collections.abc import AsyncIterator
from typing import Any
import httpx
import websockets
from websockets.asyncio.client import ClientConnection
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
from .adapter import STTAdapter
from .models import TranscriptEvent
class NvidiaSTTAdapter(STTAdapter):
"""Realtime STT adapter for NVIDIA Speech NIM."""
def __init__(
self,
server: str = "127.0.0.1:9000",
language_code: str = "pt-BR",
automatic_punctuation: bool = True,
) -> None:
self.server = server
self.language_code = language_code
self.automatic_punctuation = automatic_punctuation
self._http: httpx.AsyncClient | None = None
self._websocket: ClientConnection | None = None
self._receiver_task: asyncio.Task[None] | None = None
self._events: asyncio.Queue[TranscriptEvent | None] = asyncio.Queue()
self._started = False
self._stopping = False
self._aborted = False
self._abort_lock = asyncio.Lock()
async def start_session(self) -> None:
if self._started:
raise RuntimeError("STT session is already started.")
self._http = httpx.AsyncClient(timeout=10.0)
try:
session = await self._create_session()
self._websocket = await self._connect()
await self._initialize_session(session)
self._started = True
self._stopping = False
self._aborted = False
self._receiver_task = asyncio.create_task(
self._receive_events()
)
except Exception:
await self._cleanup()
raise
async def send_audio(self, audio: bytes) -> None:
if not self._started or self._websocket is None:
raise RuntimeError("STT session has not been started.")
if not audio:
raise ValueError("Audio chunk cannot be empty.")
await self._send_json(
{
"type": "input_audio_buffer.append",
"audio": base64.b64encode(audio).decode("utf-8"),
}
)
await self._send_json(
{
"type": "input_audio_buffer.commit",
}
)
async def stop_session(self) -> None:
if not self._started or self._websocket is None:
return
self._stopping = True
try:
await self._send_json(
{
"type": "input_audio_buffer.done",
}
)
if self._receiver_task is not None:
await self._receiver_task
finally:
await self._cleanup()
async def abort_session(self) -> None:
"""
Abort the NVIDIA transcription session safely.
Purpose:
Stop transcription immediately without waiting for a final
transcript, including when multiple abort requests overlap.
Responsibilities:
- Serialize concurrent abort requests.
- Stop accepting new audio.
- Cancel and await the receiver task.
- Release WebSocket and HTTP resources.
- Discard pending transcription events.
- Publish one end-of-stream marker.
Non-responsibilities:
- Managing Gateway WebSocket connections.
- Updating VoicePipeline turn state.
- Cancelling Agent or TTS execution.
"""
async with self._abort_lock:
# A completed abort must not run cleanup again.
if self._aborted:
return
# Mark the session as aborted before releasing resources.
self._aborted = True
self._stopping = True
self._started = False
try:
await self._cleanup()
finally:
# Remove transcripts and termination markers that may
# have been queued before the abort completed.
while True:
try:
self._events.get_nowait()
except asyncio.QueueEmpty:
break
# Publish one termination marker for the event consumer.
self._events.put_nowait(None)
def events(self) -> AsyncIterator[TranscriptEvent]:
return self._event_stream()
async def _event_stream(
self,
) -> AsyncIterator[TranscriptEvent]:
while True:
event = await self._events.get()
if event is None:
break
yield event
async def _create_session(self) -> dict[str, Any]:
if self._http is None:
raise RuntimeError("HTTP client is not initialized.")
response = await self._http.post(
f"http://{self.server}/v1/realtime/transcription_sessions"
)
response.raise_for_status()
session = response.json()
if not isinstance(session, dict):
raise RuntimeError(
"NVIDIA NIM returned an invalid session response."
)
return session
async def _connect(self) -> ClientConnection:
return await websockets.connect(
f"ws://{self.server}"
"/v1/realtime"
"?intent=transcription"
)
async def _initialize_session(
self,
session: dict[str, Any],
) -> None:
if self._websocket is None:
raise RuntimeError("NVIDIA ASR WebSocket is not connected.")
response = await self._websocket.recv()
event = self._decode_event(response)
if event.get("type") != "conversation.created":
raise RuntimeError(
"Unexpected first NIM event: "
f"{event.get('type')!r}"
)
session["input_audio_format"] = "pcm16"
input_audio_transcription = dict(
session.get("input_audio_transcription", {})
)
input_audio_transcription["language"] = self.language_code
session["input_audio_transcription"] = (
input_audio_transcription
)
session["input_audio_params"] = {
"sample_rate_hz": CANONICAL_AUDIO_FORMAT.sample_rate,
"num_channels": CANONICAL_AUDIO_FORMAT.channels,
}
recognition_config = dict(
session.get("recognition_config", {})
)
recognition_config[
"enable_automatic_punctuation"
] = self.automatic_punctuation
session["recognition_config"] = recognition_config
await self._send_json(
{
"type": "transcription_session.update",
"session": session,
}
)
response = await self._websocket.recv()
event = self._decode_event(response)
if event.get("type") != "transcription_session.updated":
raise RuntimeError(
"NVIDIA NIM did not confirm session update: "
f"{event.get('type')!r}"
)
async def _receive_events(self) -> None:
websocket = self._websocket
if websocket is None:
return
try:
async for raw_message in websocket:
# Stop processing provider events after an explicit abort.
if self._aborted:
break
event = self._decode_event(raw_message)
event_type = event.get("type", "")
if (
event_type
== "conversation.item.input_audio_transcription.delta"
):
text = event.get("delta", "")
if text:
await self._events.put(
TranscriptEvent(
type="partial",
text=text,
)
)
elif (
event_type
== "conversation.item.input_audio_transcription.completed"
):
text = event.get("transcript", "")
if not text:
continue
is_last_result = event.get(
"is_last_result",
False,
)
await self._events.put(
TranscriptEvent(
type=(
"final"
if is_last_result
else "partial"
),
text=text,
)
)
if is_last_result:
break
elif event_type.lower().startswith("error"):
error = event.get("error", {})
raise RuntimeError(
error.get(
"message",
"Unknown NVIDIA ASR error.",
)
)
except websockets.ConnectionClosed as exc:
if not self._stopping:
raise RuntimeError(
"NVIDIA ASR WebSocket closed unexpectedly."
) from exc
finally:
if not self._aborted:
await self._events.put(None)
async def _send_json(self, message: dict[str, Any]) -> None:
if self._websocket is None:
raise RuntimeError(
"NVIDIA ASR WebSocket is not connected."
)
await self._websocket.send(
json.dumps(
message,
separators=(",", ":"),
)
)
@staticmethod
def _decode_event(
message: str | bytes,
) -> dict[str, Any]:
if isinstance(message, bytes):
message = message.decode("utf-8")
try:
event = json.loads(message)
except json.JSONDecodeError as exc:
raise RuntimeError(
"Received invalid JSON from NVIDIA ASR."
) from exc
if not isinstance(event, dict):
raise RuntimeError(
"Received invalid event from NVIDIA ASR."
)
return event
async def _cleanup(self) -> None:
"""
Release NVIDIA transport resources.
Lifecycle flags are controlled by start_session(),
stop_session(), and abort_session().
"""
self._started = False
current_task = asyncio.current_task()
if self._receiver_task is not None:
if self._receiver_task is not current_task:
if not self._receiver_task.done():
self._receiver_task.cancel()
try:
await self._receiver_task
except asyncio.CancelledError:
pass
self._receiver_task = None
if self._websocket is not None:
try:
await self._websocket.close()
finally:
self._websocket = None
if self._http is not None:
await self._http.aclose()
self._http = None

9
src/voice_assistant/tts/__init__.py Normal file
View File

@@ -0,0 +1,9 @@
from .adapter import TTSAdapter
from .mock import MockTTSAdapter
from .voxcpm import VoxCPMTTSAdapter
__all__ = [
"TTSAdapter",
"MockTTSAdapter",
"VoxCPMTTSAdapter",
]

12
src/voice_assistant/tts/adapter.py Normal file
View File

@@ -0,0 +1,12 @@
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
class TTSAdapter(ABC):
@abstractmethod
async def synthesize(
self,
text: str,
) -> AsyncIterator[bytes]:
"""Convert text into audio chunks."""
raise NotImplementedError

18
src/voice_assistant/tts/mock.py Normal file
View File

@@ -0,0 +1,18 @@
from collections.abc import AsyncIterator
from .adapter import TTSAdapter
class MockTTSAdapter(TTSAdapter):
def __init__(self) -> None:
self.calls: list[str] = []
async def synthesize(
self,
text: str,
) -> AsyncIterator[bytes]:
self.calls.append(text)
yield b"\x01\x02\x03\x04"
yield b"\x05\x06\x07\x08"

57
src/voice_assistant/tts/voxcpm.py Normal file
View File

@@ -0,0 +1,57 @@
from collections.abc import AsyncIterator
import httpx
from .adapter import TTSAdapter
class VoxCPMTTSAdapter(TTSAdapter):
def __init__(
self,
server: str,
model: str,
voice: str,
response_format: str = "pcm",
stream: bool = True,
stream_format: str = "audio",
timeout: float = 120.0,
) -> None:
self.server = server.rstrip("/")
self.model = model
self.voice = voice
self.response_format = response_format
self.stream = stream
self.stream_format = stream_format
self.timeout = timeout
async def synthesize(
self,
text: str,
) -> AsyncIterator[bytes]:
if not text:
raise ValueError("Text cannot be empty.")
payload = {
"input": text,
"model": self.model,
"voice": self.voice,
"response_format": self.response_format,
"stream": self.stream,
"stream_format": self.stream_format,
}
url = f"{self.server}/v1/audio/speech"
timeout = httpx.Timeout(self.timeout)
async with httpx.AsyncClient(timeout=timeout) as client:
async with client.stream(
"POST",
url,
json=payload,
) as response:
response.raise_for_status()
async for chunk in response.aiter_bytes():
if chunk:
yield chunk

146
src/voice_assistant/turn_manager.py Normal file
View File

@@ -0,0 +1,146 @@
from dataclasses import dataclass
from enum import StrEnum
class TurnState(StrEnum):
LISTENING = "listening"
PROCESSING = "processing"
SPEAKING = "speaking"
COMPLETED = "completed"
INTERRUPTED = "interrupted"
@dataclass(slots=True)
class Turn:
turn_id: int
state: TurnState = TurnState.LISTENING
partial_text: str = ""
final_text: str | None = None
class TurnManager:
def __init__(self) -> None:
self._active_turn: Turn | None = None
@property
def active_turn(self) -> Turn | None:
return self._active_turn
def start(self, turn_id: int) -> Turn:
if self._active_turn is not None:
raise RuntimeError(
"A turn is already active."
)
self._active_turn = Turn(
turn_id=turn_id,
state=TurnState.LISTENING,
)
return self._active_turn
def on_partial(
self,
turn_id: int,
text: str,
) -> None:
turn = self._require_active_turn(turn_id)
if turn.state != TurnState.LISTENING:
raise RuntimeError(
f"Cannot receive partial transcript while "
f"turn is {turn.state.value}."
)
turn.partial_text = text
def on_final(
self,
turn_id: int,
text: str,
) -> None:
turn = self._require_active_turn(turn_id)
if turn.state != TurnState.LISTENING:
raise RuntimeError(
f"Cannot receive final transcript while "
f"turn is {turn.state.value}."
)
turn.final_text = text
turn.partial_text = text
turn.state = TurnState.PROCESSING
def on_agent_response(
self,
turn_id: int,
) -> None:
turn = self._require_active_turn(turn_id)
if turn.state != TurnState.PROCESSING:
raise RuntimeError(
f"Cannot start speaking while "
f"turn is {turn.state.value}."
)
turn.state = TurnState.SPEAKING
def on_tts_complete(
self,
turn_id: int,
) -> None:
turn = self._require_active_turn(turn_id)
if turn.state != TurnState.SPEAKING:
raise RuntimeError(
f"Cannot complete TTS while "
f"turn is {turn.state.value}."
)
turn.state = TurnState.COMPLETED
def stop(self, turn_id: int) -> Turn:
"""
Return the active turn when its complete lifecycle has finished.
A turn may only be stopped after the TTS stage has completed and
the semantic state has reached COMPLETED.
"""
turn = self._require_active_turn(turn_id)
if turn.state != TurnState.COMPLETED:
raise RuntimeError(
f"Cannot stop turn while turn is {turn.state.value}."
)
return turn
def interrupt(self, turn_id: int) -> Turn:
turn = self._require_active_turn(turn_id)
turn.state = TurnState.INTERRUPTED
return turn
def clear(self, turn_id: int) -> None:
"""
Clear the active turn after its lifecycle has been handled.
Turn identity validation is delegated to _require_active_turn().
"""
self._require_active_turn(turn_id)
self._active_turn = None
def _require_active_turn(self, turn_id: int) -> Turn:
if self._active_turn is None:
raise RuntimeError(
"There is no active turn."
)
if self._active_turn.turn_id != turn_id:
raise ValueError(
f"Unexpected turn_id={turn_id}; "
f"active_turn_id={self._active_turn.turn_id}."
)
return self._active_turn

476
src/voice_assistant/voice_pipeline.py Normal file
View File

@@ -0,0 +1,476 @@
"""
Voice turn lifecycle and provider orchestration for the Voice Assistant.
Purpose:
Coordinate session and semantic turn lifecycle and orchestrate
provider-independent Agent and TTS processing.
Responsibilities:
- Start, complete, and interrupt turns.
- Keep SessionState and TurnManager synchronized.
- Send final transcript text to the configured Agent.
- Convert the Agent response into streaming TTS audio.
- Expose provider-independent results to the transport layer.
Non-responsibilities:
- Managing WebSocket or FastAPI connections.
- Parsing WebSocket protocol messages.
- Managing provider-specific implementation details.
- Loading configuration or selecting providers.
- Sending protocol events to clients.
Architecture:
Audio Gateway
|
v
VoicePipeline
| |
v v
Session TurnManager
|
+---- AgentAdapter
|
+---- TTSAdapter
The pipeline coordinates the Voice Core while remaining independent
from WebSocket transport and concrete provider implementations.
"""
import asyncio
from collections.abc import AsyncIterator
from dataclasses import dataclass
from voice_assistant.agent.adapter import AgentAdapter
from voice_assistant.audio_gateway.app.session import SessionState
from voice_assistant.turn_manager import Turn, TurnManager
from voice_assistant.tts.adapter import TTSAdapter
DEFAULT_TURN_CANCEL_TIMEOUT = 5.0
@dataclass(slots=True)
class PipelineResult:
"""
Result produced after processing a final transcript.
The result contains only provider-independent data. Audio streaming
remains a separate operation so the transport layer can consume it
without coupling the result object to an AsyncIterator lifecycle.
"""
response_text: str
class VoicePipeline:
"""
Coordinate session, turn, Agent, and TTS processing.
Lifecycle operations are synchronous because they only update local
state. Provider execution remains asynchronous because Agent and TTS
may involve network or streaming operations.
"""
def __init__(
self,
session: SessionState,
turn_manager: TurnManager,
agent: AgentAdapter,
tts: TTSAdapter,
) -> None:
self.session = session
self.turn_manager = turn_manager
self.agent = agent
self.tts = tts
self._turn_task: asyncio.Task[None] | None = None
def start_turn(self) -> Turn:
"""
Start a new turn only when no previous execution is active.
Purpose:
Prevent overlapping voice turns after interruption
or cancellation timeout.
"""
if (
self._turn_task is not None
and not self._turn_task.done()
):
raise RuntimeError(
"Cannot start a new turn while the previous "
"turn task is still running."
)
turn_id = self.session.start_turn()
try:
return self.turn_manager.start(turn_id)
except Exception:
self.session.interrupt(turn_id)
raise
async def process_transcript(
self,
session_id: str,
turn_id: int,
text: str,
) -> PipelineResult:
"""
Process final transcript text through the configured Agent.
The turn must be in PROCESSING state before the Agent is invoked.
Once a response is produced, the semantic turn transitions to
SPEAKING so that TTS can be executed.
"""
turn = self.turn_manager.active_turn
if (
turn is None
or turn.turn_id != turn_id
):
raise ValueError(
f"Unexpected turn_id={turn_id}."
)
if turn.state.name != "PROCESSING":
raise RuntimeError(
f"Cannot process transcript while turn is "
f"{turn.state.value}."
)
response_text = await self.agent.process(
session_id=session_id,
turn_id=turn_id,
text=text,
)
self.turn_manager.on_agent_response(turn_id)
return PipelineResult(
response_text=response_text,
)
async def synthesize_response(
self,
response_text: str,
) -> AsyncIterator[bytes]:
"""
Stream TTS audio for an Agent response.
The TTS adapter remains responsible for provider-specific
synthesis and audio streaming behavior.
"""
async for audio_chunk in self.tts.synthesize(
response_text,
):
yield audio_chunk
def complete_turn(
self,
turn_id: int,
) -> Turn:
"""
Complete and release a successfully processed turn.
The TurnManager must already have reached COMPLETED through
successful TTS completion.
"""
turn = self.turn_manager.stop(turn_id)
self.turn_manager.clear(turn_id)
self.session.end_turn(turn_id)
return turn
def interrupt_turn(
self,
turn_id: int,
) -> Turn:
"""
Interrupt and release the active turn.
Actual cancellation of asynchronous provider tasks will be
added in the lifecycle/cancellation phase.
"""
turn = self.turn_manager.interrupt(turn_id)
self.session.interrupt(turn_id)
self.turn_manager.clear(turn_id)
return turn
def on_partial(
self,
turn_id: int,
text: str,
) -> None:
"""Apply a partial STT transcript to the active turn."""
self.turn_manager.on_partial(
turn_id,
text,
)
def on_final(
self,
turn_id: int,
text: str,
) -> None:
"""Apply a final STT transcript to the active turn."""
self.turn_manager.on_final(
turn_id,
text,
)
def on_tts_complete(
self,
turn_id: int,
) -> None:
"""Mark TTS completion for the active turn."""
self.turn_manager.on_tts_complete(
turn_id,
)
def on_agent_response(
self,
turn_id: int,
) -> None:
"""Mark the Agent stage as complete and start TTS."""
self.turn_manager.on_agent_response(
turn_id,
)
def close_session(self) -> None:
"""
Close the voice session and release its active turn state.
Purpose:
Ensure SessionState and TurnManager remain consistent
when the session ends, even if a turn is still active.
Responsibilities:
- Mark an active semantic turn as interrupted.
- Release the active turn from TurnManager.
- Close SessionState.
Non-responsibilities:
- Cancelling asynchronous provider tasks.
- Closing provider network connections.
- Sending WebSocket protocol events.
"""
active_turn = self.turn_manager.active_turn
if active_turn is not None:
self.turn_manager.interrupt(active_turn.turn_id)
self.turn_manager.clear(active_turn.turn_id)
self.session.close()
def is_active_turn(self, turn_id: int) -> bool:
"""
Check whether the requested turn is currently active.
Purpose:
Validate turn ownership without exposing TurnManager
internals to the Gateway.
Responsibilities:
- Verify the active semantic turn ID.
- Verify the active session turn ID.
- Return a boolean without changing state.
Non-responsibilities:
- Interrupting turns.
- Cancelling asynchronous tasks.
- Closing provider connections.
"""
active_turn = self.turn_manager.active_turn
return (
active_turn is not None
and active_turn.turn_id == turn_id
and self.session.active_turn_id == turn_id
)
def interrupt_if_active(
self,
turn_id: int,
) -> bool:
"""
Interrupt a turn only when it matches the active semantic turn.
Purpose:
Preserve the Gateway's current tolerant interruption behavior
without exposing TurnManager internals.
Returns:
True if the matching turn was interrupted.
False if no matching semantic turn exists.
Non-responsibilities:
- Cancelling asynchronous provider tasks.
- Closing provider connections.
- Sending WebSocket events.
"""
active_turn = self.turn_manager.active_turn
if (
active_turn is None
or active_turn.turn_id != turn_id
):
# Preserve the previous Gateway behavior when no matching
# semantic turn exists.
self.session.interrupt(turn_id)
return False
self.interrupt_turn(turn_id)
return True
def register_turn_task(
self,
task: asyncio.Task[None],
) -> None:
"""
Register the asynchronous execution task for the active turn.
Responsibilities:
- Associate one task with the active semantic turn.
- Reject duplicate running tasks.
- Preserve task ownership inside the VoicePipeline.
Non-responsibilities:
- Creating tasks.
- Cancelling tasks.
- Awaiting task completion.
"""
if self.turn_manager.active_turn is None:
raise RuntimeError(
"Cannot register task without an active turn."
)
if self._turn_task is not None and not self._turn_task.done():
raise RuntimeError(
"A turn execution task is already active."
)
self._turn_task = task
@property
def turn_task(self) -> asyncio.Task[None] | None:
"""Return the currently registered turn execution task."""
return self._turn_task
async def wait_turn_task(self) -> None:
"""
Await normal completion of the registered turn execution task.
Purpose:
Release task ownership after successful or failed execution
without requesting cancellation.
Responsibilities:
- Await the registered task.
- Propagate execution failures.
- Preserve cancellation of the waiting operation.
- Clear ownership only after task termination.
Non-responsibilities:
- Cancelling the task.
- Updating semantic turn state.
- Closing STT provider connections.
"""
task = self._turn_task
if task is None:
return
if task is asyncio.current_task():
raise RuntimeError(
"A turn task cannot await itself."
)
try:
await asyncio.shield(task)
finally:
if task.done() and self._turn_task is task:
self._turn_task = None
async def cancel_turn_task(
self,
timeout: float = DEFAULT_TURN_CANCEL_TIMEOUT,
) -> None:
"""
Cancel and await the registered turn execution task.
Purpose:
Bound cancellation latency without losing ownership of
a task that has not terminated.
Responsibilities:
- Reject self-cancellation.
- Request cancellation of the active task.
- Wait for termination within a configurable timeout.
- Preserve ownership if cancellation times out.
- Propagate unexpected task failures.
- Release ownership only after task termination.
Non-responsibilities:
- Updating semantic turn state.
- Closing STT provider connections.
- Sending WebSocket events.
"""
if timeout <= 0:
raise ValueError(
"Turn cancellation timeout must be positive."
)
task = self._turn_task
if task is None:
return
if task is asyncio.current_task():
raise RuntimeError(
"A turn task cannot cancel itself."
)
try:
if not task.done():
task.cancel()
try:
async with asyncio.timeout(timeout):
await asyncio.shield(task)
except TimeoutError as exc:
raise TimeoutError(
"Turn task cancellation timed out."
) from exc
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
raise
if not task.cancelled():
raise
finally:
if task.done() and self._turn_task is task:
self._turn_task = None