feat: establish voice assistant V1 baseline

This commit is contained in:
2026-10-09 18:31:21 +00:00
commit cebcb3fd61
57 changed files with 9501 additions and 0 deletions

36
.gitignore vendored Normal file
View File

@@ -0,0 +1,36 @@
# Python
__pycache__/
*.py[cod]
*.egg-info/
build/
dist/
wheels/
# Virtual environments
.venv/
venv/
# Environment variables and secrets
.env
.env.*
!.env.example
# Test and tooling caches
.pytest_cache/
.mypy_cache/
.ruff_cache/
.coverage
htmlcov/
# IDE and operating system
.vscode/
.idea/
.DS_Store
# Local audio files
audios/
*.pcm
# Temporary files
*.log
*.tmp

1
.python-version Normal file
View File

@@ -0,0 +1 @@
3.12

0
Dockerfile Normal file
View File

0
README.md Normal file
View File

18
config/development.yaml Normal file
View File

@@ -0,0 +1,18 @@
stt:
provider: nvidia
options:
server: "127.0.0.1:9000"
language_code: "pt-BR"
automatic_punctuation: true
tts:
provider: voxcpm
options:
server: "http://127.0.0.1:8000"
model: "openbmb/VoxCPM2"
voice: "default"
agent:
provider: mock
options: {}

0
config/production.yaml Normal file
View File

29
config/providers.yaml Normal file
View File

@@ -0,0 +1,29 @@
providers:
stt:
nvidia:
name: "NVIDIA ASR"
implementation: "voice_assistant.stt.nvidia:NvidiaSTTAdapter"
capabilities:
realtime: true
streaming: true
tts:
voxcpm:
name: "VoxCPM"
implementation: "voice_assistant.tts.voxcpm:VoxCPMTTSAdapter"
capabilities:
realtime: true
streaming: true
agent:
mock:
name: "Mock Agent"
implementation: "voice_assistant.agent.mock:MockAgentAdapter"
capabilities:
realtime: false
streaming: false

0
config/staging.yaml Normal file
View File

29
pyproject.toml Normal file
View File

@@ -0,0 +1,29 @@
[project]
name = "voice-assistant"
version = "0.1.0"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"fastapi>=0.141.1",
"httpx>=0.28,<1",
"httpx2>=2.13.1",
"ipykernel>=7.3.0",
"nvidia-riva-client==2.26.0",
"pytest>=9.1.1",
"python-dotenv>=1.2.3",
"requests>=2.34.2",
"uvicorn[standard]>=0.53.0",
"websockets>=15.0.1,<16",
]
[project.scripts]
voice-assistant = "voice_assistant:main"
[build-system]
requires = ["uv_build>=0.12.17,<0.13.0"]
build-backend = "uv_build"
[tool.pytest.ini_options]
testpaths = ["tests"]
pythonpath = ["src"]

View File

@@ -0,0 +1,2 @@
def main() -> None:
print("Hello from voice-assistant!")

View File

@@ -0,0 +1,7 @@
from .adapter import AgentAdapter
from .mock import MockAgentAdapter
__all__ = [
"AgentAdapter",
"MockAgentAdapter",
]

View File

@@ -0,0 +1,14 @@
from abc import ABC, abstractmethod
class AgentAdapter(ABC):
@abstractmethod
async def process(
self,
session_id: str,
turn_id: int,
text: str,
) -> str:
"""Process the final text of a turn and return the response."""
raise NotImplementedError

View File

@@ -0,0 +1,24 @@
from .adapter import AgentAdapter
class MockAgentAdapter(AgentAdapter):
def __init__(self) -> None:
self.calls: list[
tuple[str, int, str]
] = []
async def process(
self,
session_id: str,
turn_id: int,
text: str,
) -> str:
self.calls.append(
(
session_id,
turn_id,
text,
)
)
return f"Resposta simulada para: {text}"

View File

View File

@@ -0,0 +1,63 @@
"""
Canonical audio models for the Voice Assistant.
Purpose:
Define the provider-independent audio format used internally by
the Voice Core.
Responsibilities:
- Describe an audio format in a typed structure.
- Define the canonical audio format used by the Voice Core.
- Provide a shared audio contract for transports and providers.
Non-responsibilities:
- Encoding or decoding audio.
- Resampling audio.
- Converting provider-specific audio formats.
- Managing WebSocket audio frames.
- Performing STT or TTS operations.
Architecture:
Client / Transport
|
v
Canonical Audio
PCM S16LE / 16 kHz / mono
|
v
Voice Core
|
+-----+-----+
| |
v v
STT TTS
Adapter Adapter
Provider-specific format conversion, when required, belongs
inside the corresponding provider adapter.
"""
from typing import Literal
from pydantic import BaseModel
class AudioFormat(BaseModel):
"""
Describe one PCM audio format.
This model is independent from WebSocket, STT, TTS, or any
concrete provider.
"""
encoding: Literal["pcm_s16le"]
sample_rate: int
channels: int
CANONICAL_AUDIO_FORMAT = AudioFormat(
encoding="pcm_s16le",
sample_rate=16000,
channels=1,
)

View File

@@ -0,0 +1,914 @@
"""
Audio Gateway application and WebSocket transport for the Voice Assistant.
Purpose:
Expose the realtime audio protocol and connect client sessions
to the VoicePipeline orchestration layer.
Responsibilities:
- Accept and validate WebSocket protocol messages.
- Receive canonical PCM audio frames.
- Create and configure session-scoped components.
- Forward STT events and TTS audio to connected clients.
- Delegate semantic turn lifecycle to VoicePipeline.
- Coordinate transport cleanup on disconnect.
- Report protocol and cleanup errors.
Non-responsibilities:
- Implementing STT, Agent, or TTS provider logic.
- Managing semantic turn state directly.
- Loading provider implementations dynamically.
- Implementing provider-specific cancellation.
Architecture:
WebSocket Client
|
v
Audio Gateway
|
v
VoicePipeline
| | |
STT Agent TTS
"""
import json
import logging
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError
import os
import asyncio
from voice_assistant.voice_pipeline import VoicePipeline
from voice_assistant.stt.adapter import STTAdapter
from voice_assistant.tts.adapter import TTSAdapter
from voice_assistant.turn_manager import (
TurnManager,
TurnState,
)
from voice_assistant.agent.adapter import AgentAdapter
from voice_assistant.audio.models import (
CANONICAL_AUDIO_FORMAT,
)
from voice_assistant.providers import (
create_agent_adapter,
create_stt_adapter,
create_tts_adapter,
)
from voice_assistant.config.loader import (
load_config,
load_provider_catalog,
)
from voice_assistant.audio_gateway.app.protocol import (
PROTOCOL_VERSION,
AUDIO_ENCODING,
AUDIO_SAMPLE_RATE,
AUDIO_CHANNELS,
AudioChunk,
AudioStart,
AudioStop,
Interrupt,
SessionEnd,
SessionStart,
error_event,
AgentResponse,
Interrupted,
SessionEnded,
SessionReady,
STTFinal,
STTPartial,
TurnEnded,
TurnStarted,
)
from voice_assistant.audio_gateway.app.session import SessionState
# logger configuration
logger = logging.getLogger(__name__)
app = FastAPI(
title="Voice Assistant Audio Gateway",
version=PROTOCOL_VERSION,
)
app_config = load_config()
provider_catalog = load_provider_catalog()
MAX_AUDIO_CHUNK_BYTES = 64 * 1024 # 64 KB
def create_turn_manager() -> TurnManager:
"""Create the turn manager used by the audio gateway."""
return TurnManager()
async def send_json(
websocket: WebSocket,
payload: BaseModel | dict,
) -> None:
"""
Serialize and send one protocol event through the WebSocket.
Typed Pydantic protocol models are preferred. Dictionary support
remains temporarily available for error events until ErrorEvent
is migrated to the typed protocol.
"""
if isinstance(payload, BaseModel):
data = payload.model_dump()
else:
data = payload
await websocket.send_text(
json.dumps(
data,
separators=(",", ":"),
)
)
async def forward_stt_events(
websocket: WebSocket,
stt: STTAdapter,
pipeline: VoicePipeline,
session_id: str,
turn_id: int,
) -> None:
async for event in stt.events():
if not pipeline.is_active_turn(turn_id):
logger.debug(
"Discarding stale STT event. session_id=%s turn_id=%s",
session_id,
turn_id,
)
break
if event.type == "partial":
pipeline.on_partial(
turn_id,
event.text,
)
await send_json(
websocket,
STTPartial(
session_id=session_id,
turn_id=turn_id,
text=event.text,
),
)
elif event.type == "final":
pipeline.on_final(
turn_id,
event.text,
)
await send_json(
websocket,
STTFinal(
session_id=session_id,
turn_id=turn_id,
text=event.text,
),
)
result = await pipeline.process_transcript(
session_id=session_id,
turn_id=turn_id,
text=event.text,
)
await send_json(
websocket,
AgentResponse(
session_id=session_id,
turn_id=turn_id,
text=result.response_text,
),
)
async for audio_chunk in pipeline.synthesize_response(
result.response_text,
):
await websocket.send_bytes(audio_chunk)
pipeline.on_tts_complete(turn_id)
@app.get("/health")
async def health() -> dict[str, str]:
return {"status": "ok"}
@app.get("/ready")
async def ready() -> dict[str, str]:
return {"status": "ready"}
@app.websocket("/ws/audio")
async def audio_socket(websocket: WebSocket) -> None:
await websocket.accept()
session: SessionState | None = None
pending_audio_chunk: AudioChunk | None = None
stt: STTAdapter | None = None
stt_events_task: asyncio.Task | None = None
turn_manager = create_turn_manager()
agent: AgentAdapter | None = None
tts: TTSAdapter | None = None
pipeline: VoicePipeline | None = None
try:
while True:
message = await websocket.receive()
if message.get("type") == "websocket.disconnect":
break
# ---------------------------------------------------------
# Binary WebSocket frame -> PCM
# ---------------------------------------------------------
if message.get("bytes") is not None:
if session is None:
await send_json(
websocket,
error_event(
"SESSION_NOT_FOUND",
"session.start is required before audio.",
),
)
continue
if pending_audio_chunk is None:
await send_json(
websocket,
error_event(
"INVALID_AUDIO_FRAME",
"Binary PCM frame must be preceded by audio.chunk.",
session_id=session.session_id,
turn_id=session.active_turn_id,
),
)
continue
audio_payload = message["bytes"]
if len(audio_payload) > MAX_AUDIO_CHUNK_BYTES:
await send_json(
websocket,
error_event(
"MESSAGE_TOO_LARGE",
(
"Audio chunk exceeds maximum size of "
f"{MAX_AUDIO_CHUNK_BYTES} bytes."
),
session_id=session.session_id,
turn_id=pending_audio_chunk.turn_id,
details={
"max_bytes": MAX_AUDIO_CHUNK_BYTES,
"received_bytes": len(audio_payload),
},
),
)
pending_audio_chunk = None
continue
try:
session.accept_audio_chunk(
pending_audio_chunk.turn_id,
pending_audio_chunk.sequence_number,
len(audio_payload),
)
except (RuntimeError, ValueError) as exc:
await send_json(
websocket,
error_event(
"AUDIO_SEQUENCE_ERROR",
str(exc),
session_id=session.session_id,
turn_id=pending_audio_chunk.turn_id,
details={
"expected_sequence_number":
session.expected_audio_sequence,
},
),
)
pending_audio_chunk = None
continue
# echo do PCM para validar o transporte.
if stt is None:
await send_json(
websocket,
error_event(
"STT_NOT_STARTED",
"audio.start is required before audio.",
session_id=session.session_id,
turn_id=session.active_turn_id,
),
)
pending_audio_chunk = None
continue
try:
await stt.send_audio(audio_payload)
except (RuntimeError, ValueError) as exc:
await send_json(
websocket,
error_event(
"STT_AUDIO_ERROR",
str(exc),
session_id=session.session_id,
turn_id=pending_audio_chunk.turn_id,
retryable=True,
),
)
pending_audio_chunk = None
continue
pending_audio_chunk = None
continue
# ---------------------------------------------------------
# Text WebSocket frame -> JSON control event
# ---------------------------------------------------------
text = message.get("text")
if text is None:
await send_json(
websocket,
error_event(
"INVALID_MESSAGE",
"Unsupported WebSocket frame.",
),
)
continue
try:
payload = json.loads(text)
except json.JSONDecodeError:
await send_json(
websocket,
error_event(
"INVALID_MESSAGE",
"Message must contain valid JSON.",
),
)
continue
event_type = payload.get("type")
if not isinstance(event_type, str):
await send_json(
websocket,
error_event(
"INVALID_MESSAGE",
"Field 'type' is required.",
),
)
continue
# ---------------------------------------------------------
# session.start
# ---------------------------------------------------------
if event_type == "session.start":
if session is not None:
await send_json(
websocket,
error_event(
"SESSION_ALREADY_STARTED",
"This WebSocket already has an active session.",
session_id=session.session_id,
),
)
continue
try:
request = SessionStart.model_validate(payload)
except ValidationError as exc:
await send_json(
websocket,
error_event(
"INVALID_MESSAGE",
"Invalid session.start payload.",
details={"errors": exc.errors()},
),
)
continue
audio = request.audio
if (
audio.encoding != AUDIO_ENCODING
or audio.sample_rate != AUDIO_SAMPLE_RATE
or audio.channels != AUDIO_CHANNELS
):
await send_json(
websocket,
error_event(
"INVALID_AUDIO_FORMAT",
(
"Only PCM S16LE, mono, 16 kHz "
"is supported in the MVP."
),
details={
"expected": {
"encoding": AUDIO_ENCODING,
"sample_rate": AUDIO_SAMPLE_RATE,
"channels": AUDIO_CHANNELS,
}
},
),
)
continue
session = SessionState(
session_id=request.session_id or SessionState().session_id
)
agent = create_agent_adapter(
config=app_config,
catalog=provider_catalog,
)
tts = create_tts_adapter(
config=app_config,
catalog=provider_catalog,
)
pipeline = VoicePipeline(
session=session,
turn_manager=turn_manager,
agent=agent,
tts=tts,
)
await send_json(
websocket,
SessionReady(
protocol_version=PROTOCOL_VERSION,
session_id=session.session_id,
audio=CANONICAL_AUDIO_FORMAT,
),
)
continue
# ---------------------------------------------------------
# Todos os eventos abaixo exigem sessão ativa
# ---------------------------------------------------------
if session.closed:
await send_json(
websocket,
error_event(
"SESSION_ALREADY_CLOSED",
"This session is already closed.",
session_id=session.session_id,
),
)
continue
# ---------------------------------------------------------
# audio.start
# ---------------------------------------------------------
if event_type == "audio.start":
turn_id: int | None = None
try:
if pipeline is None:
raise RuntimeError(
"Voice pipeline is not initialized."
)
turn = pipeline.start_turn()
turn_id = turn.turn_id
# Each turn receives a fresh STT provider instance.
stt = create_stt_adapter(
config=app_config,
catalog=provider_catalog,
)
await stt.start_session()
stt_events_task = asyncio.create_task(
forward_stt_events(
websocket=websocket,
stt=stt,
pipeline=pipeline,
session_id=session.session_id,
turn_id=turn_id,
)
)
pipeline.register_turn_task(stt_events_task)
except (ValidationError, RuntimeError, ValueError) as exc:
# Preserve the original startup failure.
startup_error = exc
cleanup_complete = True
# Cancel and await any task created during startup.
if stt_events_task is not None:
task = stt_events_task
try:
if (
pipeline is not None
and pipeline.turn_task is task
):
await pipeline.cancel_turn_task()
else:
if not task.done():
task.cancel()
try:
await asyncio.wait_for(
asyncio.shield(task),
timeout=5.0,
)
except asyncio.CancelledError:
if not task.cancelled():
raise
except Exception:
cleanup_complete = False
logger.exception(
"Failed to cancel task during "
"audio.start rollback."
)
if task.done():
stt_events_task = None
else:
cleanup_complete = False
# Abort the partially initialized STT provider.
if stt is not None:
try:
await stt.abort_session()
except Exception:
cleanup_complete = False
logger.exception(
"Failed to abort STT during "
"audio.start rollback."
)
# Only release semantic state after execution
# has stopped and provider cleanup succeeded.
if cleanup_complete:
if turn_id is not None and pipeline is not None:
pipeline.interrupt_turn(turn_id)
stt = None
pending_audio_chunk = None
await send_json(
websocket,
error_event(
"STT_START_FAILED",
str(startup_error),
session_id=session.session_id,
turn_id=turn_id,
retryable=cleanup_complete,
),
)
continue
# Startup completed successfully.
await send_json(
websocket,
TurnStarted(
session_id=session.session_id,
turn_id=turn_id,
),
)
continue
# ---------------------------------------------------------
# audio.chunk
# ---------------------------------------------------------
if event_type == "audio.chunk":
try:
request = AudioChunk.model_validate(payload)
if session.active_turn_id != request.turn_id:
raise ValueError(
f"Unexpected turn_id={request.turn_id}; "
f"active_turn_id={session.active_turn_id}"
)
if pending_audio_chunk is not None:
raise ValueError(
"Previous audio chunk is still pending PCM data."
)
pending_audio_chunk = request
except (ValidationError, ValueError) as exc:
await send_json(
websocket,
error_event(
"AUDIO_SEQUENCE_ERROR",
str(exc),
session_id=session.session_id,
turn_id=payload.get("turn_id"),
details={
"expected_sequence_number":
session.expected_audio_sequence,
},
),
)
continue
continue
# ---------------------------------------------------------
# audio.stop
# ---------------------------------------------------------
if event_type == "audio.stop":
try:
request = AudioStop.model_validate(payload)
if stt is None:
raise RuntimeError(
"STT session has not been started."
)
if pending_audio_chunk is not None:
raise ValueError(
"audio.stop received while an audio chunk is pending."
)
if agent is None:
raise RuntimeError(
"Agent adapter is not initialized."
)
if tts is None:
raise RuntimeError(
"TTS adapter is not initialized."
)
if pipeline is None:
raise RuntimeError(
"Voice pipeline is not initialized."
)
# Finish the STT session normally.
await stt.stop_session()
await pipeline.wait_turn_task()
stt_events_task = None
try:
pipeline.complete_turn(request.turn_id)
except (RuntimeError, ValueError) as exc:
raise RuntimeError(
"Turn pipeline did not complete successfully."
) from exc
except (ValidationError, RuntimeError, ValueError) as exc:
await send_json(
websocket,
error_event(
"INVALID_TURN",
str(exc),
session_id=session.session_id,
turn_id=payload.get("turn_id"),
),
)
continue
stt = None
pending_audio_chunk = None
await send_json(
websocket,
TurnEnded(
session_id=session.session_id,
turn_id=request.turn_id,
),
)
continue
# ---------------------------------------------------------
# interrupt
# ---------------------------------------------------------
if event_type == "interrupt":
try:
request = Interrupt.model_validate(payload)
except ValidationError as exc:
await send_json(
websocket,
error_event(
"INVALID_MESSAGE",
"Invalid interrupt payload.",
session_id=session.session_id,
details={"errors": exc.errors()},
),
)
continue
if pipeline is None:
await send_json(
websocket,
error_event(
"INVALID_TURN",
"Voice pipeline is not initialized.",
session_id=session.session_id,
turn_id=request.turn_id,
),
)
continue
# Only cancel resources belonging to the requested active turn.
if pipeline.is_active_turn(request.turn_id):
try:
# Cancel and await STT/Agent/TTS execution before
# confirming the interruption to the client.
await pipeline.cancel_turn_task()
# Abort transcription without requesting a final result.
if stt is not None:
await stt.abort_session()
# Release semantic turn state after execution stops.
pipeline.interrupt_if_active(request.turn_id)
# Clear references to the interrupted execution.
stt_events_task = None
pending_audio_chunk = None
except Exception as exc:
await send_json(
websocket,
error_event(
"TURN_CANCEL_FAILED",
str(exc),
session_id=session.session_id,
turn_id=request.turn_id,
),
)
continue
else:
# Preserve the existing tolerant behavior for unknown turns.
pipeline.interrupt_if_active(request.turn_id)
await send_json(
websocket,
Interrupted(
session_id=session.session_id,
turn_id=request.turn_id,
),
)
continue
# ---------------------------------------------------------
# session.end
# ---------------------------------------------------------
if event_type == "session.end":
try:
SessionEnd.model_validate(payload)
if pipeline is None:
raise RuntimeError(
"Voice pipeline is not initialized."
)
# Cancel and await any running STT/Agent/TTS task.
await pipeline.cancel_turn_task()
# Abort STT without waiting for a final transcript.
if stt is not None:
await stt.abort_session()
# Close semantic turn and session state.
pipeline.close_session()
# Release Gateway-owned references.
stt_events_task = None
stt = None
pending_audio_chunk = None
except (ValidationError, RuntimeError, ValueError) as exc:
await send_json(
websocket,
error_event(
"SESSION_END_FAILED",
str(exc),
session_id=session.session_id,
turn_id=session.active_turn_id,
retryable=True,
),
)
continue
await send_json(
websocket,
SessionEnded(
session_id=session.session_id,
),
)
break
# ---------------------------------------------------------
# Evento desconhecido
# ---------------------------------------------------------
await send_json(
websocket,
error_event(
"UNSUPPORTED_EVENT",
f"Unsupported event type: {event_type!r}.",
session_id=session.session_id,
turn_id=session.active_turn_id,
),
)
except WebSocketDisconnect:
# The finally block owns connection cleanup.
pass
finally:
# Cancel and await the task owned by VoicePipeline.
if pipeline is not None:
try:
await pipeline.cancel_turn_task()
except Exception:
logger.exception(
"Failed to cancel voice turn task. session_id=%s",
session.session_id if session else None,
)
# A task may have been created before registration failed.
# Handle it independently if the pipeline does not own it.
if stt_events_task is not None:
task = stt_events_task
if (
pipeline is None
or pipeline.turn_task is not task
):
try:
if not task.done():
task.cancel()
try:
await asyncio.wait_for(
asyncio.shield(task),
timeout=5.0,
)
except asyncio.CancelledError:
if not task.cancelled():
raise
except TimeoutError:
logger.error(
"Unregistered STT task cleanup timed out. "
"session_id=%s",
session.session_id if session else None,
)
except Exception:
logger.exception(
"Failed to clean up unregistered STT task. "
"session_id=%s",
session.session_id if session else None,
)
if task.done():
stt_events_task = None
# Abort STT without waiting for a final transcript.
if stt is not None:
try:
await stt.abort_session()
except Exception:
logger.exception(
"Failed to abort STT session. session_id=%s",
session.session_id if session else None,
)
# Release session and semantic turn state.
try:
if pipeline is not None:
pipeline.close_session()
elif session is not None:
session.close()
except Exception:
logger.exception(
"Failed to close voice session. session_id=%s",
session.session_id if session else None,
)

View File

@@ -0,0 +1,163 @@
from __future__ import annotations
from typing import Any, Literal
from pydantic import BaseModel, Field
from voice_assistant.audio.models import (
AudioFormat,
CANONICAL_AUDIO_FORMAT,
)
PROTOCOL_VERSION = "1.0"
AUDIO_ENCODING = "pcm_s16le"
AUDIO_SAMPLE_RATE = 16000
AUDIO_CHANNELS = 1
class SessionStart(BaseModel):
"""Request that initializes a new Voice Assistant session."""
type: Literal["session.start"] = "session.start"
session_id: str | None = None
audio: AudioFormat = Field(
default_factory=lambda: CANONICAL_AUDIO_FORMAT.model_copy()
)
class ErrorEvent(BaseModel):
"""Error response sent by the Audio Gateway."""
type: Literal["error"] = "error"
session_id: str | None = None
turn_id: int | None = None
code: str
message: str
retryable: bool = False
details: dict[str, Any] = Field(
default_factory=dict,
)
class AudioStart(BaseModel):
"""Request that starts a new audio turn."""
type: Literal["audio.start"] = "audio.start"
class AudioChunk(BaseModel):
"""Metadata describing the next binary audio chunk."""
type: Literal["audio.chunk"] = "audio.chunk"
turn_id: int
sequence_number: int
class AudioStop(BaseModel):
"""Request that finishes audio input for the active turn."""
type: Literal["audio.stop"] = "audio.stop"
turn_id: int
class Interrupt(BaseModel):
"""Request that interrupts the active turn."""
type: Literal["interrupt"] = "interrupt"
turn_id: int
class SessionEnd(BaseModel):
"""Request that closes the active session."""
type: Literal["session.end"] = "session.end"
class SessionReady(BaseModel):
"""Response confirming that a session is ready."""
type: Literal["session.ready"] = "session.ready"
protocol_version: str
session_id: str
audio: AudioFormat
class TurnStarted(BaseModel):
"""Response confirming that a new turn has started."""
type: Literal["turn.started"] = "turn.started"
session_id: str
turn_id: int
class STTPartial(BaseModel):
"""Partial transcription produced during a turn."""
type: Literal["stt.partial"] = "stt.partial"
session_id: str
turn_id: int
text: str
class STTFinal(BaseModel):
"""Final transcription produced for a turn."""
type: Literal["stt.final"] = "stt.final"
session_id: str
turn_id: int
text: str
class AgentResponse(BaseModel):
"""Text response produced by the configured Agent."""
type: Literal["agent.response"] = "agent.response"
session_id: str
turn_id: int
text: str
class TurnEnded(BaseModel):
"""Response confirming successful completion of a turn."""
type: Literal["turn.ended"] = "turn.ended"
session_id: str
turn_id: int
class Interrupted(BaseModel):
"""Response confirming interruption of a turn."""
type: Literal["interrupted"] = "interrupted"
session_id: str
turn_id: int
class SessionEnded(BaseModel):
"""Response confirming that a session has ended."""
type: Literal["session.ended"] = "session.ended"
session_id: str
def error_event(
code: str,
message: str,
*,
session_id: str | None = None,
turn_id: int | None = None,
retryable: bool = False,
details: dict[str, Any] | None = None,
) -> ErrorEvent:
"""Build a typed error protocol event."""
return ErrorEvent(
session_id=session_id,
turn_id=turn_id,
code=code,
message=message,
retryable=retryable,
details=details or {},
)

View File

@@ -0,0 +1,70 @@
from dataclasses import dataclass, field
from uuid import uuid4
@dataclass
class SessionState:
session_id: str = field(default_factory=lambda: str(uuid4()))
active_turn_id: int | None = None
next_turn_id: int = 1
expected_audio_sequence: int = 0
closed: bool = False
def start_turn(self) -> int:
if self.closed:
raise RuntimeError("Session is closed.")
if self.active_turn_id is not None:
raise RuntimeError("A turn is already active.")
self.active_turn_id = self.next_turn_id
self.next_turn_id += 1
self.expected_audio_sequence = 0
return self.active_turn_id
def accept_audio_chunk(
self,
turn_id: int,
sequence_number: int,
payload_size: int,
) -> None:
if self.closed:
raise RuntimeError("Session is closed.")
if self.active_turn_id != turn_id:
raise ValueError(
f"Unexpected turn_id={turn_id}; "
f"active_turn_id={self.active_turn_id}."
)
if sequence_number != self.expected_audio_sequence:
raise ValueError(
f"Expected sequence_number="
f"{self.expected_audio_sequence}, "
f"received={sequence_number}."
)
if payload_size == 0:
raise ValueError("Audio chunk cannot be empty.")
self.expected_audio_sequence += 1
def end_turn(self, turn_id: int) -> None:
if self.active_turn_id != turn_id:
raise ValueError(
f"Cannot end turn {turn_id}; "
f"active_turn_id={self.active_turn_id}."
)
self.active_turn_id = None
self.expected_audio_sequence = 0
def interrupt(self, turn_id: int) -> None:
if self.active_turn_id == turn_id:
self.active_turn_id = None
self.expected_audio_sequence = 0
def close(self) -> None:
self.active_turn_id = None
self.closed = True

View File

View File

@@ -0,0 +1,109 @@
"""
Configuration loading for the Voice Assistant.
Purpose:
Load environment-specific YAML configuration and the provider
catalog, then validate both using the typed configuration models.
Responsibilities:
- Select the active environment.
- Load environment configuration.
- Load provider catalog metadata.
- Validate configuration through Pydantic models.
Non-responsibilities:
- Instantiating providers.
- Executing STT, Agent, or TTS functionality.
- Managing application lifecycle.
"""
import os
from pathlib import Path
from typing import Any
import yaml
from .models import AppConfig, ProviderCatalog
DEFAULT_CONFIG_DIR = Path("config")
def get_environment() -> str:
"""
Return the configured application environment.
Defaults to development for local execution.
"""
return os.getenv(
"APP_ENV",
"development",
)
def load_config(
environment: str | None = None,
config_dir: Path = DEFAULT_CONFIG_DIR,
) -> AppConfig:
"""Load and validate the selected environment configuration."""
environment = environment or get_environment()
config_path = config_dir / f"{environment}.yaml"
if not config_path.exists():
raise FileNotFoundError(
f"Configuration file not found: {config_path}"
)
with config_path.open(
"r",
encoding="utf-8",
) as file:
data: Any = yaml.safe_load(file)
if data is None:
raise ValueError(
f"Configuration file is empty: {config_path}"
)
if not isinstance(data, dict):
raise ValueError(
f"Configuration root must be a mapping: "
f"{config_path}"
)
return AppConfig.model_validate(data)
def load_provider_catalog(
config_dir: Path = DEFAULT_CONFIG_DIR,
) -> ProviderCatalog:
"""Load and validate the provider catalog."""
config_path = config_dir / "providers.yaml"
if not config_path.exists():
raise FileNotFoundError(
f"Provider catalog not found: {config_path}"
)
with config_path.open(
"r",
encoding="utf-8",
) as file:
data: Any = yaml.safe_load(file)
if data is None:
raise ValueError(
f"Provider catalog is empty: {config_path}"
)
if not isinstance(data, dict):
raise ValueError(
f"Provider catalog root must be a mapping: "
f"{config_path}"
)
return ProviderCatalog.model_validate(data)

View File

@@ -0,0 +1,73 @@
"""
Typed configuration models for the Voice Assistant.
Purpose:
Define and validate the configuration structures used by the
application and by the dynamic provider loading system.
Responsibilities:
- Define the application configuration for STT, Agent, and TTS.
- Define the catalog structure for available providers.
- Represent provider-specific options without coupling the core
configuration model to a specific vendor or implementation.
- Describe provider capabilities for discovery and future tooling.
Non-responsibilities:
- Loading YAML files.
- Reading environment variables or secrets.
- Selecting or instantiating provider implementations.
- Executing STT, Agent, or TTS functionality.
Architecture:
YAML
|
v
Config Loader
|
v
AppConfig / ProviderCatalog
|
v
Provider Loader
|
v
STTAdapter / AgentAdapter / TTSAdapter
"""
from typing import Any
from pydantic import BaseModel, Field
class ProviderDefinition(BaseModel):
"""Metadata describing one available provider."""
name: str
implementation: str
capabilities: dict[str, bool] = Field(
default_factory=dict,
)
class ProviderCatalog(BaseModel):
"""Available providers grouped by component type."""
providers: dict[
str,
dict[str, ProviderDefinition],
] = Field(default_factory=dict)
class ComponentConfig(BaseModel):
"""Selected provider and its provider-specific runtime options."""
provider: str
options: dict[str, Any] = Field(default_factory=dict)
class AppConfig(BaseModel):
"""Provider selections for one application environment."""
stt: ComponentConfig
tts: ComponentConfig
agent: ComponentConfig

View File

@@ -0,0 +1,187 @@
"""
Dynamic provider loading and composition for the Voice Assistant.
Purpose:
Resolve provider names from the application configuration and
instantiate implementations that satisfy the corresponding adapter
contract.
Responsibilities:
- Resolve provider implementations from ProviderCatalog metadata.
- Import provider classes dynamically.
- Validate that the loaded implementation matches the expected adapter.
- Instantiate STT, Agent, and TTS providers using provider-specific options.
Non-responsibilities:
- Loading YAML files.
- Reading secrets directly.
- Managing WebSocket connections.
- Managing turn state.
- Executing provider functionality.
Architecture:
AppConfig + ProviderCatalog
|
v
Provider Loader
|
v
STTAdapter / AgentAdapter / TTSAdapter
|
v
concrete provider
"""
from importlib import import_module
from typing import Any
from voice_assistant.agent.adapter import AgentAdapter
from voice_assistant.config.models import (
AppConfig,
ComponentConfig,
ProviderCatalog,
)
from voice_assistant.stt.adapter import STTAdapter
from voice_assistant.tts.adapter import TTSAdapter
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
PROVIDER_CONTRACTS: dict[str, type] = {
"stt": STTAdapter,
"agent": AgentAdapter,
"tts": TTSAdapter,
}
def load_provider_class(
catalog: ProviderCatalog,
component: str,
provider: str,
) -> type:
"""
Load and validate a provider class from the provider catalog.
The catalog stores the import path, while this function ensures that
the resolved class implements the adapter contract associated with the
requested component.
"""
contract = PROVIDER_CONTRACTS.get(component)
if contract is None:
raise ValueError(
f"Unsupported provider component: {component!r}."
)
component_providers = catalog.providers.get(component)
if component_providers is None:
raise ValueError(
f"No providers are registered for component {component!r}."
)
definition = component_providers.get(provider)
if definition is None:
raise ValueError(
f"Provider {provider!r} is not registered for "
f"component {component!r}."
)
module_name, class_name = definition.implementation.split(
":",
maxsplit=1,
)
try:
module = import_module(module_name)
except ImportError as exc:
raise RuntimeError(
f"Could not import provider module "
f"{module_name!r} for {component!r}/{provider!r}."
) from exc
try:
provider_class = getattr(module, class_name)
except AttributeError as exc:
raise RuntimeError(
f"Provider class {class_name!r} was not found in "
f"module {module_name!r}."
) from exc
if not isinstance(provider_class, type):
raise TypeError(
f"Provider {component!r}/{provider!r} does not resolve "
f"to a class."
)
if not issubclass(provider_class, contract):
raise TypeError(
f"Provider {component!r}/{provider!r} must implement "
f"{contract.__name__}."
)
return provider_class
def _create_provider(
catalog: ProviderCatalog,
component: str,
config: ComponentConfig,
) -> Any:
"""
Instantiate the provider selected by a component configuration.
Provider-specific constructor arguments are kept inside `options`
so the common configuration model does not need to know the API of
every vendor.
"""
provider_class = load_provider_class(
catalog=catalog,
component=component,
provider=config.provider,
)
try:
return provider_class(**config.options)
except TypeError as exc:
raise TypeError(
f"Invalid options for provider "
f"{component!r}/{config.provider!r}: {exc}"
) from exc
def create_stt_adapter(
config: AppConfig,
catalog: ProviderCatalog,
) -> STTAdapter:
"""Create the configured STT adapter."""
return _create_provider(
catalog=catalog,
component="stt",
config=config.stt,
)
def create_agent_adapter(
config: AppConfig,
catalog: ProviderCatalog,
) -> AgentAdapter:
"""Create the configured Agent adapter."""
return _create_provider(
catalog=catalog,
component="agent",
config=config.agent,
)
def create_tts_adapter(
config: AppConfig,
catalog: ProviderCatalog,
) -> TTSAdapter:
"""Create the configured TTS adapter."""
return _create_provider(
catalog=catalog,
component="tts",
config=config.tts,
)

View File

View File

@@ -0,0 +1,85 @@
"""
Provider-independent Speech-to-Text adapter contract.
Purpose:
Define the interface that every STT provider must implement
to participate in the Voice Assistant pipeline.
Responsibilities:
- Start transcription sessions.
- Receive canonical PCM audio chunks.
- Finish transcription sessions normally.
- Abort transcription sessions during interruption or cleanup.
- Expose provider-independent transcript events.
Non-responsibilities:
- Managing Audio Gateway WebSocket connections.
- Managing SessionState or TurnManager.
- Executing Agent or TTS operations.
- Implementing provider-specific communication.
Architecture:
Audio Gateway / VoicePipeline
|
v
STTAdapter
|
+-------+-------+
| |
v v
NvidiaSTTAdapter MockSTTAdapter
Concrete providers implement this contract without exposing
their internal protocols to the Voice Core.
"""
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from .models import TranscriptEvent
class STTAdapter(ABC):
"""Abstract contract for Speech-to-Text providers."""
@abstractmethod
async def start_session(self) -> None:
"""Initialize a transcription session."""
raise NotImplementedError
@abstractmethod
async def send_audio(self, audio: bytes) -> None:
"""Send one canonical PCM audio chunk for transcription."""
raise NotImplementedError
@abstractmethod
async def stop_session(self) -> None:
"""
Finish transcription normally.
The provider may wait for its final transcription result
before releasing the session.
"""
raise NotImplementedError
@abstractmethod
async def abort_session(self) -> None:
"""
Abort transcription without waiting for a final result.
Implementations must:
- Stop accepting new audio.
- Release active provider resources.
- Unblock consumers waiting for transcript events.
- Avoid waiting for final transcription.
- Support repeated calls safely.
The provider must not emit new transcription results after
the abort operation has completed.
"""
raise NotImplementedError
@abstractmethod
def events(self) -> AsyncIterator[TranscriptEvent]:
"""Expose the asynchronous stream of transcript events."""
raise NotImplementedError

View File

@@ -0,0 +1,161 @@
"""
In-memory mock STT provider for the Voice Assistant.
Purpose:
Simulate a realtime STT provider without external services.
Responsibilities:
- Simulate transcription session lifecycle.
- Validate incoming audio chunks.
- Expose partial and final transcript events.
- Support normal stop and immediate abort.
- Provide deterministic behavior for automated tests.
Non-responsibilities:
- Performing real speech recognition.
- Managing network connections.
- Managing voice turns or Gateway sessions.
- Executing Agent or TTS operations.
Architecture:
Tests / VoicePipeline
|
v
STTAdapter
|
v
MockSTTAdapter
|
v
Async event queue
"""
import asyncio
from collections.abc import AsyncIterator
from .adapter import STTAdapter
from .models import TranscriptEvent
class MockSTTAdapter(STTAdapter):
def __init__(self) -> None:
self._events: asyncio.Queue[
TranscriptEvent | None
] = asyncio.Queue()
self._started = False
async def start_session(self) -> None:
self._started = True
self._events = asyncio.Queue()
async def send_audio(self, audio: bytes) -> None:
if not self._started:
raise RuntimeError(
"STT session has not been started."
)
if not audio:
raise ValueError(
"Audio chunk cannot be empty."
)
async def stop_session(self) -> None:
if not self._started:
return
self._started = False
await self._events.put(None)
async def abort_session(self) -> None:
"""
Abort transcription and discard pending transcript events.
Unlike stop_session(), abort_session() must not deliver
queued partial or final transcription results.
"""
if not self._started:
return
# Reject further audio and transcript events.
self._started = False
# Discard queued transcript events from the aborted session.
while True:
try:
self._events.get_nowait()
except asyncio.QueueEmpty:
break
# Unblock consumers waiting for the next event.
await self._events.put(None)
def events(self) -> AsyncIterator[TranscriptEvent]:
return self._event_stream()
async def _event_stream(
self,
) -> AsyncIterator[TranscriptEvent]:
while True:
event = await self._events.get()
if event is None:
break
yield event
async def emit_partial(self, text: str) -> None:
if not self._started:
raise RuntimeError(
"STT session has not been started."
)
await self._events.put(
TranscriptEvent(
type="partial",
text=text,
)
)
async def emit_final(self, text: str) -> None:
if not self._started:
raise RuntimeError(
"STT session has not been started."
)
await self._events.put(
TranscriptEvent(
type="final",
text=text,
)
)
def emit_partial_now(self, text: str) -> None:
if not self._started:
raise RuntimeError(
"STT session has not been started."
)
self._events.put_nowait(
TranscriptEvent(
type="partial",
text=text,
)
)
def emit_final_now(self, text: str) -> None:
if not self._started:
raise RuntimeError(
"STT session has not been started."
)
self._events.put_nowait(
TranscriptEvent(
type="final",
text=text,
)
)

View File

@@ -0,0 +1,11 @@
from dataclasses import dataclass
from typing import Literal
TranscriptType = Literal["partial", "final"]
@dataclass(slots=True)
class TranscriptEvent:
type: TranscriptType
text: str

View File

@@ -0,0 +1,430 @@
"""
NVIDIA realtime STT provider for the Voice Assistant.
Purpose:
Implement the generic STTAdapter contract using NVIDIA Speech NIM.
Responsibilities:
- Create and manage NVIDIA realtime transcription sessions.
- Send canonical PCM audio to the NVIDIA ASR service.
- Receive provider-specific transcription events.
- Translate NVIDIA events into provider-independent TranscriptEvent objects.
- Manage provider connection lifecycle and cleanup.
Non-responsibilities:
- Managing WebSocket client connections to the Audio Gateway.
- Managing Voice Assistant sessions or turns.
- Executing Agent or TTS functionality.
- Defining the canonical Voice Core audio format.
Architecture:
Voice Core
|
| canonical PCM audio
v
STTAdapter
|
v
NvidiaSTTAdapter
|
v
NVIDIA Speech NIM
NVIDIA-specific HTTP, WebSocket, session, and event details remain
isolated inside this adapter.
"""
from __future__ import annotations
import asyncio
import base64
import json
from collections.abc import AsyncIterator
from typing import Any
import httpx
import websockets
from websockets.asyncio.client import ClientConnection
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
from .adapter import STTAdapter
from .models import TranscriptEvent
class NvidiaSTTAdapter(STTAdapter):
"""Realtime STT adapter for NVIDIA Speech NIM."""
def __init__(
self,
server: str = "127.0.0.1:9000",
language_code: str = "pt-BR",
automatic_punctuation: bool = True,
) -> None:
self.server = server
self.language_code = language_code
self.automatic_punctuation = automatic_punctuation
self._http: httpx.AsyncClient | None = None
self._websocket: ClientConnection | None = None
self._receiver_task: asyncio.Task[None] | None = None
self._events: asyncio.Queue[TranscriptEvent | None] = asyncio.Queue()
self._started = False
self._stopping = False
self._aborted = False
self._abort_lock = asyncio.Lock()
async def start_session(self) -> None:
if self._started:
raise RuntimeError("STT session is already started.")
self._http = httpx.AsyncClient(timeout=10.0)
try:
session = await self._create_session()
self._websocket = await self._connect()
await self._initialize_session(session)
self._started = True
self._stopping = False
self._aborted = False
self._receiver_task = asyncio.create_task(
self._receive_events()
)
except Exception:
await self._cleanup()
raise
async def send_audio(self, audio: bytes) -> None:
if not self._started or self._websocket is None:
raise RuntimeError("STT session has not been started.")
if not audio:
raise ValueError("Audio chunk cannot be empty.")
await self._send_json(
{
"type": "input_audio_buffer.append",
"audio": base64.b64encode(audio).decode("utf-8"),
}
)
await self._send_json(
{
"type": "input_audio_buffer.commit",
}
)
async def stop_session(self) -> None:
if not self._started or self._websocket is None:
return
self._stopping = True
try:
await self._send_json(
{
"type": "input_audio_buffer.done",
}
)
if self._receiver_task is not None:
await self._receiver_task
finally:
await self._cleanup()
async def abort_session(self) -> None:
"""
Abort the NVIDIA transcription session safely.
Purpose:
Stop transcription immediately without waiting for a final
transcript, including when multiple abort requests overlap.
Responsibilities:
- Serialize concurrent abort requests.
- Stop accepting new audio.
- Cancel and await the receiver task.
- Release WebSocket and HTTP resources.
- Discard pending transcription events.
- Publish one end-of-stream marker.
Non-responsibilities:
- Managing Gateway WebSocket connections.
- Updating VoicePipeline turn state.
- Cancelling Agent or TTS execution.
"""
async with self._abort_lock:
# A completed abort must not run cleanup again.
if self._aborted:
return
# Mark the session as aborted before releasing resources.
self._aborted = True
self._stopping = True
self._started = False
try:
await self._cleanup()
finally:
# Remove transcripts and termination markers that may
# have been queued before the abort completed.
while True:
try:
self._events.get_nowait()
except asyncio.QueueEmpty:
break
# Publish one termination marker for the event consumer.
self._events.put_nowait(None)
def events(self) -> AsyncIterator[TranscriptEvent]:
return self._event_stream()
async def _event_stream(
self,
) -> AsyncIterator[TranscriptEvent]:
while True:
event = await self._events.get()
if event is None:
break
yield event
async def _create_session(self) -> dict[str, Any]:
if self._http is None:
raise RuntimeError("HTTP client is not initialized.")
response = await self._http.post(
f"http://{self.server}/v1/realtime/transcription_sessions"
)
response.raise_for_status()
session = response.json()
if not isinstance(session, dict):
raise RuntimeError(
"NVIDIA NIM returned an invalid session response."
)
return session
async def _connect(self) -> ClientConnection:
return await websockets.connect(
f"ws://{self.server}"
"/v1/realtime"
"?intent=transcription"
)
async def _initialize_session(
self,
session: dict[str, Any],
) -> None:
if self._websocket is None:
raise RuntimeError("NVIDIA ASR WebSocket is not connected.")
response = await self._websocket.recv()
event = self._decode_event(response)
if event.get("type") != "conversation.created":
raise RuntimeError(
"Unexpected first NIM event: "
f"{event.get('type')!r}"
)
session["input_audio_format"] = "pcm16"
input_audio_transcription = dict(
session.get("input_audio_transcription", {})
)
input_audio_transcription["language"] = self.language_code
session["input_audio_transcription"] = (
input_audio_transcription
)
session["input_audio_params"] = {
"sample_rate_hz": CANONICAL_AUDIO_FORMAT.sample_rate,
"num_channels": CANONICAL_AUDIO_FORMAT.channels,
}
recognition_config = dict(
session.get("recognition_config", {})
)
recognition_config[
"enable_automatic_punctuation"
] = self.automatic_punctuation
session["recognition_config"] = recognition_config
await self._send_json(
{
"type": "transcription_session.update",
"session": session,
}
)
response = await self._websocket.recv()
event = self._decode_event(response)
if event.get("type") != "transcription_session.updated":
raise RuntimeError(
"NVIDIA NIM did not confirm session update: "
f"{event.get('type')!r}"
)
async def _receive_events(self) -> None:
websocket = self._websocket
if websocket is None:
return
try:
async for raw_message in websocket:
# Stop processing provider events after an explicit abort.
if self._aborted:
break
event = self._decode_event(raw_message)
event_type = event.get("type", "")
if (
event_type
== "conversation.item.input_audio_transcription.delta"
):
text = event.get("delta", "")
if text:
await self._events.put(
TranscriptEvent(
type="partial",
text=text,
)
)
elif (
event_type
== "conversation.item.input_audio_transcription.completed"
):
text = event.get("transcript", "")
if not text:
continue
is_last_result = event.get(
"is_last_result",
False,
)
await self._events.put(
TranscriptEvent(
type=(
"final"
if is_last_result
else "partial"
),
text=text,
)
)
if is_last_result:
break
elif event_type.lower().startswith("error"):
error = event.get("error", {})
raise RuntimeError(
error.get(
"message",
"Unknown NVIDIA ASR error.",
)
)
except websockets.ConnectionClosed as exc:
if not self._stopping:
raise RuntimeError(
"NVIDIA ASR WebSocket closed unexpectedly."
) from exc
finally:
if not self._aborted:
await self._events.put(None)
async def _send_json(self, message: dict[str, Any]) -> None:
if self._websocket is None:
raise RuntimeError(
"NVIDIA ASR WebSocket is not connected."
)
await self._websocket.send(
json.dumps(
message,
separators=(",", ":"),
)
)
@staticmethod
def _decode_event(
message: str | bytes,
) -> dict[str, Any]:
if isinstance(message, bytes):
message = message.decode("utf-8")
try:
event = json.loads(message)
except json.JSONDecodeError as exc:
raise RuntimeError(
"Received invalid JSON from NVIDIA ASR."
) from exc
if not isinstance(event, dict):
raise RuntimeError(
"Received invalid event from NVIDIA ASR."
)
return event
async def _cleanup(self) -> None:
"""
Release NVIDIA transport resources.
Lifecycle flags are controlled by start_session(),
stop_session(), and abort_session().
"""
self._started = False
current_task = asyncio.current_task()
if self._receiver_task is not None:
if self._receiver_task is not current_task:
if not self._receiver_task.done():
self._receiver_task.cancel()
try:
await self._receiver_task
except asyncio.CancelledError:
pass
self._receiver_task = None
if self._websocket is not None:
try:
await self._websocket.close()
finally:
self._websocket = None
if self._http is not None:
await self._http.aclose()
self._http = None

View File

@@ -0,0 +1,9 @@
from .adapter import TTSAdapter
from .mock import MockTTSAdapter
from .voxcpm import VoxCPMTTSAdapter
__all__ = [
"TTSAdapter",
"MockTTSAdapter",
"VoxCPMTTSAdapter",
]

View File

@@ -0,0 +1,12 @@
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
class TTSAdapter(ABC):
@abstractmethod
async def synthesize(
self,
text: str,
) -> AsyncIterator[bytes]:
"""Convert text into audio chunks."""
raise NotImplementedError

View File

@@ -0,0 +1,18 @@
from collections.abc import AsyncIterator
from .adapter import TTSAdapter
class MockTTSAdapter(TTSAdapter):
def __init__(self) -> None:
self.calls: list[str] = []
async def synthesize(
self,
text: str,
) -> AsyncIterator[bytes]:
self.calls.append(text)
yield b"\x01\x02\x03\x04"
yield b"\x05\x06\x07\x08"

View File

@@ -0,0 +1,57 @@
from collections.abc import AsyncIterator
import httpx
from .adapter import TTSAdapter
class VoxCPMTTSAdapter(TTSAdapter):
def __init__(
self,
server: str,
model: str,
voice: str,
response_format: str = "pcm",
stream: bool = True,
stream_format: str = "audio",
timeout: float = 120.0,
) -> None:
self.server = server.rstrip("/")
self.model = model
self.voice = voice
self.response_format = response_format
self.stream = stream
self.stream_format = stream_format
self.timeout = timeout
async def synthesize(
self,
text: str,
) -> AsyncIterator[bytes]:
if not text:
raise ValueError("Text cannot be empty.")
payload = {
"input": text,
"model": self.model,
"voice": self.voice,
"response_format": self.response_format,
"stream": self.stream,
"stream_format": self.stream_format,
}
url = f"{self.server}/v1/audio/speech"
timeout = httpx.Timeout(self.timeout)
async with httpx.AsyncClient(timeout=timeout) as client:
async with client.stream(
"POST",
url,
json=payload,
) as response:
response.raise_for_status()
async for chunk in response.aiter_bytes():
if chunk:
yield chunk

View File

@@ -0,0 +1,146 @@
from dataclasses import dataclass
from enum import StrEnum
class TurnState(StrEnum):
LISTENING = "listening"
PROCESSING = "processing"
SPEAKING = "speaking"
COMPLETED = "completed"
INTERRUPTED = "interrupted"
@dataclass(slots=True)
class Turn:
turn_id: int
state: TurnState = TurnState.LISTENING
partial_text: str = ""
final_text: str | None = None
class TurnManager:
def __init__(self) -> None:
self._active_turn: Turn | None = None
@property
def active_turn(self) -> Turn | None:
return self._active_turn
def start(self, turn_id: int) -> Turn:
if self._active_turn is not None:
raise RuntimeError(
"A turn is already active."
)
self._active_turn = Turn(
turn_id=turn_id,
state=TurnState.LISTENING,
)
return self._active_turn
def on_partial(
self,
turn_id: int,
text: str,
) -> None:
turn = self._require_active_turn(turn_id)
if turn.state != TurnState.LISTENING:
raise RuntimeError(
f"Cannot receive partial transcript while "
f"turn is {turn.state.value}."
)
turn.partial_text = text
def on_final(
self,
turn_id: int,
text: str,
) -> None:
turn = self._require_active_turn(turn_id)
if turn.state != TurnState.LISTENING:
raise RuntimeError(
f"Cannot receive final transcript while "
f"turn is {turn.state.value}."
)
turn.final_text = text
turn.partial_text = text
turn.state = TurnState.PROCESSING
def on_agent_response(
self,
turn_id: int,
) -> None:
turn = self._require_active_turn(turn_id)
if turn.state != TurnState.PROCESSING:
raise RuntimeError(
f"Cannot start speaking while "
f"turn is {turn.state.value}."
)
turn.state = TurnState.SPEAKING
def on_tts_complete(
self,
turn_id: int,
) -> None:
turn = self._require_active_turn(turn_id)
if turn.state != TurnState.SPEAKING:
raise RuntimeError(
f"Cannot complete TTS while "
f"turn is {turn.state.value}."
)
turn.state = TurnState.COMPLETED
def stop(self, turn_id: int) -> Turn:
"""
Return the active turn when its complete lifecycle has finished.
A turn may only be stopped after the TTS stage has completed and
the semantic state has reached COMPLETED.
"""
turn = self._require_active_turn(turn_id)
if turn.state != TurnState.COMPLETED:
raise RuntimeError(
f"Cannot stop turn while turn is {turn.state.value}."
)
return turn
def interrupt(self, turn_id: int) -> Turn:
turn = self._require_active_turn(turn_id)
turn.state = TurnState.INTERRUPTED
return turn
def clear(self, turn_id: int) -> None:
"""
Clear the active turn after its lifecycle has been handled.
Turn identity validation is delegated to _require_active_turn().
"""
self._require_active_turn(turn_id)
self._active_turn = None
def _require_active_turn(self, turn_id: int) -> Turn:
if self._active_turn is None:
raise RuntimeError(
"There is no active turn."
)
if self._active_turn.turn_id != turn_id:
raise ValueError(
f"Unexpected turn_id={turn_id}; "
f"active_turn_id={self._active_turn.turn_id}."
)
return self._active_turn

View File

@@ -0,0 +1,476 @@
"""
Voice turn lifecycle and provider orchestration for the Voice Assistant.
Purpose:
Coordinate session and semantic turn lifecycle and orchestrate
provider-independent Agent and TTS processing.
Responsibilities:
- Start, complete, and interrupt turns.
- Keep SessionState and TurnManager synchronized.
- Send final transcript text to the configured Agent.
- Convert the Agent response into streaming TTS audio.
- Expose provider-independent results to the transport layer.
Non-responsibilities:
- Managing WebSocket or FastAPI connections.
- Parsing WebSocket protocol messages.
- Managing provider-specific implementation details.
- Loading configuration or selecting providers.
- Sending protocol events to clients.
Architecture:
Audio Gateway
|
v
VoicePipeline
| |
v v
Session TurnManager
|
+---- AgentAdapter
|
+---- TTSAdapter
The pipeline coordinates the Voice Core while remaining independent
from WebSocket transport and concrete provider implementations.
"""
import asyncio
from collections.abc import AsyncIterator
from dataclasses import dataclass
from voice_assistant.agent.adapter import AgentAdapter
from voice_assistant.audio_gateway.app.session import SessionState
from voice_assistant.turn_manager import Turn, TurnManager
from voice_assistant.tts.adapter import TTSAdapter
DEFAULT_TURN_CANCEL_TIMEOUT = 5.0
@dataclass(slots=True)
class PipelineResult:
"""
Result produced after processing a final transcript.
The result contains only provider-independent data. Audio streaming
remains a separate operation so the transport layer can consume it
without coupling the result object to an AsyncIterator lifecycle.
"""
response_text: str
class VoicePipeline:
"""
Coordinate session, turn, Agent, and TTS processing.
Lifecycle operations are synchronous because they only update local
state. Provider execution remains asynchronous because Agent and TTS
may involve network or streaming operations.
"""
def __init__(
self,
session: SessionState,
turn_manager: TurnManager,
agent: AgentAdapter,
tts: TTSAdapter,
) -> None:
self.session = session
self.turn_manager = turn_manager
self.agent = agent
self.tts = tts
self._turn_task: asyncio.Task[None] | None = None
def start_turn(self) -> Turn:
"""
Start a new turn only when no previous execution is active.
Purpose:
Prevent overlapping voice turns after interruption
or cancellation timeout.
"""
if (
self._turn_task is not None
and not self._turn_task.done()
):
raise RuntimeError(
"Cannot start a new turn while the previous "
"turn task is still running."
)
turn_id = self.session.start_turn()
try:
return self.turn_manager.start(turn_id)
except Exception:
self.session.interrupt(turn_id)
raise
async def process_transcript(
self,
session_id: str,
turn_id: int,
text: str,
) -> PipelineResult:
"""
Process final transcript text through the configured Agent.
The turn must be in PROCESSING state before the Agent is invoked.
Once a response is produced, the semantic turn transitions to
SPEAKING so that TTS can be executed.
"""
turn = self.turn_manager.active_turn
if (
turn is None
or turn.turn_id != turn_id
):
raise ValueError(
f"Unexpected turn_id={turn_id}."
)
if turn.state.name != "PROCESSING":
raise RuntimeError(
f"Cannot process transcript while turn is "
f"{turn.state.value}."
)
response_text = await self.agent.process(
session_id=session_id,
turn_id=turn_id,
text=text,
)
self.turn_manager.on_agent_response(turn_id)
return PipelineResult(
response_text=response_text,
)
async def synthesize_response(
self,
response_text: str,
) -> AsyncIterator[bytes]:
"""
Stream TTS audio for an Agent response.
The TTS adapter remains responsible for provider-specific
synthesis and audio streaming behavior.
"""
async for audio_chunk in self.tts.synthesize(
response_text,
):
yield audio_chunk
def complete_turn(
self,
turn_id: int,
) -> Turn:
"""
Complete and release a successfully processed turn.
The TurnManager must already have reached COMPLETED through
successful TTS completion.
"""
turn = self.turn_manager.stop(turn_id)
self.turn_manager.clear(turn_id)
self.session.end_turn(turn_id)
return turn
def interrupt_turn(
self,
turn_id: int,
) -> Turn:
"""
Interrupt and release the active turn.
Actual cancellation of asynchronous provider tasks will be
added in the lifecycle/cancellation phase.
"""
turn = self.turn_manager.interrupt(turn_id)
self.session.interrupt(turn_id)
self.turn_manager.clear(turn_id)
return turn
def on_partial(
self,
turn_id: int,
text: str,
) -> None:
"""Apply a partial STT transcript to the active turn."""
self.turn_manager.on_partial(
turn_id,
text,
)
def on_final(
self,
turn_id: int,
text: str,
) -> None:
"""Apply a final STT transcript to the active turn."""
self.turn_manager.on_final(
turn_id,
text,
)
def on_tts_complete(
self,
turn_id: int,
) -> None:
"""Mark TTS completion for the active turn."""
self.turn_manager.on_tts_complete(
turn_id,
)
def on_agent_response(
self,
turn_id: int,
) -> None:
"""Mark the Agent stage as complete and start TTS."""
self.turn_manager.on_agent_response(
turn_id,
)
def close_session(self) -> None:
"""
Close the voice session and release its active turn state.
Purpose:
Ensure SessionState and TurnManager remain consistent
when the session ends, even if a turn is still active.
Responsibilities:
- Mark an active semantic turn as interrupted.
- Release the active turn from TurnManager.
- Close SessionState.
Non-responsibilities:
- Cancelling asynchronous provider tasks.
- Closing provider network connections.
- Sending WebSocket protocol events.
"""
active_turn = self.turn_manager.active_turn
if active_turn is not None:
self.turn_manager.interrupt(active_turn.turn_id)
self.turn_manager.clear(active_turn.turn_id)
self.session.close()
def is_active_turn(self, turn_id: int) -> bool:
"""
Check whether the requested turn is currently active.
Purpose:
Validate turn ownership without exposing TurnManager
internals to the Gateway.
Responsibilities:
- Verify the active semantic turn ID.
- Verify the active session turn ID.
- Return a boolean without changing state.
Non-responsibilities:
- Interrupting turns.
- Cancelling asynchronous tasks.
- Closing provider connections.
"""
active_turn = self.turn_manager.active_turn
return (
active_turn is not None
and active_turn.turn_id == turn_id
and self.session.active_turn_id == turn_id
)
def interrupt_if_active(
self,
turn_id: int,
) -> bool:
"""
Interrupt a turn only when it matches the active semantic turn.
Purpose:
Preserve the Gateway's current tolerant interruption behavior
without exposing TurnManager internals.
Returns:
True if the matching turn was interrupted.
False if no matching semantic turn exists.
Non-responsibilities:
- Cancelling asynchronous provider tasks.
- Closing provider connections.
- Sending WebSocket events.
"""
active_turn = self.turn_manager.active_turn
if (
active_turn is None
or active_turn.turn_id != turn_id
):
# Preserve the previous Gateway behavior when no matching
# semantic turn exists.
self.session.interrupt(turn_id)
return False
self.interrupt_turn(turn_id)
return True
def register_turn_task(
self,
task: asyncio.Task[None],
) -> None:
"""
Register the asynchronous execution task for the active turn.
Responsibilities:
- Associate one task with the active semantic turn.
- Reject duplicate running tasks.
- Preserve task ownership inside the VoicePipeline.
Non-responsibilities:
- Creating tasks.
- Cancelling tasks.
- Awaiting task completion.
"""
if self.turn_manager.active_turn is None:
raise RuntimeError(
"Cannot register task without an active turn."
)
if self._turn_task is not None and not self._turn_task.done():
raise RuntimeError(
"A turn execution task is already active."
)
self._turn_task = task
@property
def turn_task(self) -> asyncio.Task[None] | None:
"""Return the currently registered turn execution task."""
return self._turn_task
async def wait_turn_task(self) -> None:
"""
Await normal completion of the registered turn execution task.
Purpose:
Release task ownership after successful or failed execution
without requesting cancellation.
Responsibilities:
- Await the registered task.
- Propagate execution failures.
- Preserve cancellation of the waiting operation.
- Clear ownership only after task termination.
Non-responsibilities:
- Cancelling the task.
- Updating semantic turn state.
- Closing STT provider connections.
"""
task = self._turn_task
if task is None:
return
if task is asyncio.current_task():
raise RuntimeError(
"A turn task cannot await itself."
)
try:
await asyncio.shield(task)
finally:
if task.done() and self._turn_task is task:
self._turn_task = None
async def cancel_turn_task(
self,
timeout: float = DEFAULT_TURN_CANCEL_TIMEOUT,
) -> None:
"""
Cancel and await the registered turn execution task.
Purpose:
Bound cancellation latency without losing ownership of
a task that has not terminated.
Responsibilities:
- Reject self-cancellation.
- Request cancellation of the active task.
- Wait for termination within a configurable timeout.
- Preserve ownership if cancellation times out.
- Propagate unexpected task failures.
- Release ownership only after task termination.
Non-responsibilities:
- Updating semantic turn state.
- Closing STT provider connections.
- Sending WebSocket events.
"""
if timeout <= 0:
raise ValueError(
"Turn cancellation timeout must be positive."
)
task = self._turn_task
if task is None:
return
if task is asyncio.current_task():
raise RuntimeError(
"A turn task cannot cancel itself."
)
try:
if not task.done():
task.cancel()
try:
async with asyncio.timeout(timeout):
await asyncio.shield(task)
except TimeoutError as exc:
raise TimeoutError(
"Turn task cancellation timed out."
) from exc
except asyncio.CancelledError:
current = asyncio.current_task()
if current is not None and current.cancelling():
raise
if not task.cancelled():
raise
finally:
if task.done() and self._turn_task is task:
self._turn_task = None

40
tests/test_agent.py Normal file
View File

@@ -0,0 +1,40 @@
import pytest
from voice_assistant.agent import MockAgentAdapter
@pytest.mark.anyio
async def text_mock_agent_process_text():
agent = MackAgentAdapter()
response = await agent.process(
session_id="session-1",
turn_id=1,
text="Qual o horário de funcionamento?",
)
assert response == (
"Resposta simulada para: "
"Qual o horário de funcionamento?"
)
@pytest.mark.anyio
async def test_mock_agent_records_call():
agent = MockAgentAdapter()
await agent.process(
session_id="session-1",
turn_id=7,
text= "Olá",
)
assert agent.calls == [
(
"session-1",
7,
"Olá",
)
]

View File

@@ -0,0 +1,31 @@
"""
Tests for the canonical Voice Assistant audio contract.
Purpose:
Verify that the Voice Core exposes the expected provider-independent
canonical audio format.
"""
import pytest
from pydantic import ValidationError
from voice_assistant.audio.models import (
AudioFormat,
CANONICAL_AUDIO_FORMAT,
)
def test_canonical_audio_format():
assert CANONICAL_AUDIO_FORMAT.encoding == "pcm_s16le"
assert CANONICAL_AUDIO_FORMAT.sample_rate == 16000
assert CANONICAL_AUDIO_FORMAT.channels == 1
def test_audio_format_rejects_unsupported_encoding():
with pytest.raises(ValidationError):
AudioFormat(
encoding="mp3",
sample_rate=16000,
channels=1,
)

288
tests/test_config.py Normal file
View File

@@ -0,0 +1,288 @@
"""
Typed configuration models for the Voice Assistant.
Purpose:
Define and validate the configuration structures used by the
application and dynamic provider loading system.
Responsibilities:
- Define the application configuration for STT, Agent, and TTS.
- Define the provider catalog structure.
- Represent provider-specific options without coupling the core
configuration model to a specific vendor or implementation.
- Describe provider capabilities for discovery and future tooling.
Non-responsibilities:
- Loading YAML files.
- Reading environment variables or secrets.
- Selecting or instantiating provider implementations.
- Executing STT, Agent, or TTS functionality.
Architecture:
YAML
|
v
Config Loader
|
v
AppConfig / ProviderCatalog
|
v
Provider Loader
|
v
STTAdapter / AgentAdapter / TTSAdapter
"""
import pytest
import yaml
from voice_assistant.config.loader import load_config
from voice_assistant.config.loader import (
load_config,
load_provider_catalog,
)
def test_load_development_config():
config = load_config()
assert config.stt.provider == "nvidia"
assert config.stt.options["server"] == "127.0.0.1:9000"
assert config.stt.options["language_code"] == "pt-BR"
assert config.stt.options["automatic_punctuation"] is True
assert config.tts.provider == "voxcpm"
assert config.tts.options["server"] == "http://127.0.0.1:8000"
assert config.tts.options["model"] == "openbmb/VoxCPM2"
assert config.tts.options["voice"] == "default"
assert config.agent.provider == "mock"
def test_load_config_missing_file(tmp_path):
with pytest.raises(
FileNotFoundError,
match="Configuration file not found",
):
load_config(
environment="development",
config_dir=tmp_path,
)
def test_load_config_empty_file(tmp_path):
config_file = tmp_path / "development.yaml"
config_file.write_text(
"",
encoding="utf-8",
)
with pytest.raises(
ValueError,
match="Configuration file is empty",
):
load_config(
environment="development",
config_dir=tmp_path,
)
def test_load_config_root_must_be_mapping(tmp_path):
config_file = tmp_path / "development.yaml"
config_file.write_text(
"- item1\n- item2\n",
encoding="utf-8",
)
with pytest.raises(
ValueError,
match="Configuration root must be a mapping",
):
load_config(
environment="development",
config_dir=tmp_path,
)
def test_load_config_validates_required_fields(tmp_path):
config_file = tmp_path / "development.yaml"
config_file.write_text(
yaml.safe_dump(
{
"stt": {
"provider": "nvidia",
"server": "127.0.0.1:9000",
"language": "pt-BR",
},
"tts": {
"provider": "voxcpm",
"server": "http://127.0.0.1:8000",
"model": "openbmb/VoxCPM2",
"voice": "default",
},
# agent intentionally missing
}
),
encoding="utf-8",
)
with pytest.raises(
Exception,
match="agent",
):
load_config(
environment="development",
config_dir=tmp_path,
)
def test_load_provider_catalog():
catalog = load_provider_catalog()
assert "nvidia" in catalog.providers["stt"]
assert "voxcpm" in catalog.providers["tts"]
assert "mock" in catalog.providers["agent"]
voxcpm = catalog.providers["tts"]["voxcpm"]
assert voxcpm.name == "VoxCPM"
assert (
voxcpm.implementation
== "voice_assistant.tts.voxcpm:VoxCPMTTSAdapter"
)
assert voxcpm.capabilities == {
"realtime": True,
"streaming": True,
}
def test_load_provider_catalog_missing_file(tmp_path):
with pytest.raises(
FileNotFoundError,
match="Provider catalog not found",
):
load_provider_catalog(
config_dir=tmp_path,
)
def test_load_config_uses_app_env(
tmp_path,
monkeypatch,
):
development = tmp_path / "development.yaml"
development.write_text(
"""
stt:
provider: nvidia
options:
server: "dev-asr"
tts:
provider: voxcpm
options:
server: "dev-tts"
model: "dev-model"
voice: "default"
agent:
provider: mock
options: {}
""",
encoding="utf-8",
)
monkeypatch.setenv(
"APP_ENV",
"development",
)
config = load_config(
config_dir=tmp_path,
)
assert config.stt.options["server"] == "dev-asr"
assert config.tts.options["server"] == "dev-tts"
assert config.tts.options["model"] == "dev-model"
def test_load_config_rejects_unknown_environment(
tmp_path,
):
with pytest.raises(
FileNotFoundError,
match="Configuration file not found",
):
load_config(
environment="does-not-exist",
config_dir=tmp_path,
)
def test_load_config_uses_app_env(
tmp_path,
monkeypatch,
):
config_file = tmp_path / "staging.yaml"
config_file.write_text(
"""
stt:
provider: nvidia
options:
server: "staging-asr:9000"
tts:
provider: voxcpm
options:
server: "http://staging-tts:8000"
model: "staging-model"
voice: "staging-voice"
agent:
provider: mock
options: {}
""",
encoding="utf-8",
)
monkeypatch.setenv(
"APP_ENV",
"staging",
)
config = load_config(
config_dir=tmp_path,
)
assert config.stt.provider == "nvidia"
assert (
config.stt.options["server"]
== "staging-asr:9000"
)
assert config.tts.provider == "voxcpm"
assert (
config.tts.options["server"]
== "http://staging-tts:8000"
)
assert config.tts.options["model"] == "staging-model"
assert config.tts.options["voice"] == "staging-voice"
assert config.agent.provider == "mock"
def test_load_config_rejects_unknown_environment(
tmp_path,
):
with pytest.raises(
FileNotFoundError,
match="Configuration file not found",
):
load_config(
environment="does-not-exist",
config_dir=tmp_path,
)

1701
tests/test_gateway.py Normal file

File diff suppressed because it is too large Load Diff

0
tests/test_nvidia_stt.py Normal file
View File

View File

@@ -0,0 +1,71 @@
import asyncio
import wave
from voice_assistant.stt.nvidia import NvidiaSTTAdapter
AUDIO_FILE = (
"/home/ubuntu/voice-assistant/audios/chafik_normalize.wav"
)
CHUNK_FRAMES = 1600
async def main() -> None:
stt = NvidiaSTTAdapter(
server="127.0.0.1:9000",
language_code="pt-BR",
automatic_punctuation=True,
)
print("Starting STT session...")
await stt.start_session()
print("STT session ready.")
async def send_audio() -> None:
with wave.open(AUDIO_FILE, "rb") as wav:
while True:
audio = wav.readframes(CHUNK_FRAMES)
if not audio:
break
await stt.send_audio(audio)
async def receive_events() -> None:
async for event in stt.events():
print(
f"[{event.type.upper():7}] {event.text}"
)
sender = asyncio.create_task(send_audio())
receiver = asyncio.create_task(receive_events())
try:
await sender
print("Audio stream sent.")
await stt.stop_session()
await receiver
finally:
if not sender.done():
sender.cancel()
if not receiver.done():
receiver.cancel()
await asyncio.gather(
sender,
receiver,
return_exceptions=True,
)
print("STT session finished.")
if __name__ == "__main__":
asyncio.run(main())

170
tests/test_protocol.py Normal file
View File

@@ -0,0 +1,170 @@
"""
Tests for the Voice Assistant WebSocket protocol.
Purpose:
Verify that protocol message models enforce their declared
event types and canonical audio contract.
Responsibilities:
- Validate inbound protocol event discriminators.
- Validate SessionStart audio configuration.
- Reject messages whose type does not match their model.
Non-responsibilities:
- Testing WebSocket transport behavior.
- Testing session or turn lifecycle.
- Testing STT, Agent, or TTS providers.
"""
import pytest
from pydantic import ValidationError
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
from voice_assistant.audio_gateway.app.protocol import (
AgentResponse,
AudioChunk,
AudioStart,
AudioStop,
ErrorEvent,
Interrupt,
Interrupted,
SessionEnd,
SessionEnded,
SessionReady,
SessionStart,
STTFinal,
STTPartial,
TurnEnded,
TurnStarted,
)
@pytest.mark.parametrize(
("model", "expected_type"),
[
(SessionStart, "session.start"),
(AudioStart, "audio.start"),
(AudioChunk, "audio.chunk"),
(AudioStop, "audio.stop"),
(Interrupt, "interrupt"),
(SessionEnd, "session.end"),
],
)
def test_protocol_event_has_expected_type(
model,
expected_type,
):
payload = {
"type": expected_type,
}
if model in (AudioChunk, AudioStop, Interrupt):
payload["turn_id"] = 1
if model is AudioChunk:
payload["sequence_number"] = 0
event = model.model_validate(payload)
assert event.type == expected_type
def test_protocol_rejects_wrong_event_type():
with pytest.raises(ValidationError):
AudioStart.model_validate(
{
"type": "audio.stop",
}
)
def test_session_ready_output():
event = SessionReady(
protocol_version="1.0",
session_id="session-1",
audio=CANONICAL_AUDIO_FORMAT,
)
assert event.model_dump() == {
"type": "session.ready",
"protocol_version": "1.0",
"session_id": "session-1",
"audio": {
"encoding": "pcm_s16le",
"sample_rate": 16000,
"channels": 1,
},
}
@pytest.mark.parametrize(
("model", "event_type"),
[
(TurnStarted, "turn.started"),
(STTPartial, "stt.partial"),
(STTFinal, "stt.final"),
(AgentResponse, "agent.response"),
(TurnEnded, "turn.ended"),
(Interrupted, "interrupted"),
],
)
def test_turn_output_event_types(
model,
event_type,
):
data = {
"session_id": "session-1",
"turn_id": 1,
}
if model in (STTPartial, STTFinal, AgentResponse):
data["text"] = "test"
event = model(**data)
assert event.type == event_type
def test_session_ended_output():
event = SessionEnded(
session_id="session-1",
)
assert event.type == "session.ended"
assert event.session_id == "session-1"
def test_error_event():
event = ErrorEvent(
session_id="session-1",
turn_id=1,
code="TEST_ERROR",
message="Something went wrong.",
retryable=True,
details={
"reason": "test",
},
)
assert event.model_dump() == {
"type": "error",
"session_id": "session-1",
"turn_id": 1,
"code": "TEST_ERROR",
"message": "Something went wrong.",
"retryable": True,
"details": {
"reason": "test",
},
}
def test_error_event_defaults():
event = ErrorEvent(
code="TEST_ERROR",
message="Something went wrong.",
)
assert event.type == "error"
assert event.session_id is None
assert event.turn_id is None
assert event.retryable is False
assert event.details == {}

View File

@@ -0,0 +1,83 @@
"""
Provider factory tests for the Voice Assistant.
Purpose:
Verify that the provider composition layer resolves the configured
providers dynamically and passes the configured options correctly.
Responsibilities:
- Validate STT provider creation.
- Validate Agent provider creation.
- Validate TTS provider creation.
- Verify provider-specific options are propagated to each adapter.
Non-responsibilities:
- Testing provider internals.
- Testing WebSocket behavior.
- Testing external services.
Architecture:
AppConfig + ProviderCatalog
|
v
provider factories
|
v
concrete Adapter instances
"""
from voice_assistant.agent.mock import MockAgentAdapter
from voice_assistant.config.loader import (
load_config,
load_provider_catalog,
)
from voice_assistant.providers import (
create_agent_adapter,
create_stt_adapter,
create_tts_adapter,
)
from voice_assistant.stt.nvidia import NvidiaSTTAdapter
from voice_assistant.tts.voxcpm import VoxCPMTTSAdapter
def test_create_stt_adapter_from_config():
config = load_config()
catalog = load_provider_catalog()
stt = create_stt_adapter(
config=config,
catalog=catalog,
)
assert isinstance(stt, NvidiaSTTAdapter)
assert stt.server == "127.0.0.1:9000"
assert stt.language_code == "pt-BR"
assert stt.automatic_punctuation is True
def test_create_agent_adapter_from_config():
config = load_config()
catalog = load_provider_catalog()
agent = create_agent_adapter(
config=config,
catalog=catalog,
)
assert isinstance(agent, MockAgentAdapter)
def test_create_tts_adapter_from_config():
config = load_config()
catalog = load_provider_catalog()
tts = create_tts_adapter(
config=config,
catalog=catalog,
)
assert isinstance(tts, VoxCPMTTSAdapter)
assert tts.server == "http://127.0.0.1:8000"
assert tts.model == "openbmb/VoxCPM2"
assert tts.voice == "default"

75
tests/test_providers.py Normal file
View File

@@ -0,0 +1,75 @@
import pytest
from voice_assistant.agent.mock import MockAgentAdapter
from voice_assistant.config.loader import load_provider_catalog
from voice_assistant.tts.voxcpm import VoxCPMTTSAdapter
from voice_assistant.config.models import ProviderCatalog
from voice_assistant.providers import load_provider_class
def test_provider_must_implement_component_contract():
catalog = ProviderCatalog.model_validate(
{
"providers": {
"tts": {
"invalid": {
"name": "Invalid TTS Provider",
"implementation": (
"voice_assistant.agent.mock:"
"MockAgentAdapter"
),
"capabilities": {},
}
}
}
}
)
with pytest.raises(
TypeError,
match="must implement TTSAdapter",
):
load_provider_class(
catalog=catalog,
component="tts",
provider="invalid",
)
def test_load_tts_provider_class():
catalog = load_provider_catalog()
provider_class = load_provider_class(
catalog=catalog,
component="tts",
provider="voxcpm",
)
assert provider_class is VoxCPMTTSAdapter
def test_load_agent_provider_class():
catalog = load_provider_catalog()
provider_class = load_provider_class(
catalog=catalog,
component="agent",
provider="mock",
)
assert provider_class is MockAgentAdapter
def test_unknown_provider_fails():
catalog = load_provider_catalog()
with pytest.raises(
ValueError,
match="is not registered",
):
load_provider_class(
catalog=catalog,
component="tts",
provider="does-not-exist",
)

View File

@@ -0,0 +1,281 @@
"""
Tests for SessionState and TurnManager lifecycle ownership.
Purpose:
Define and verify the separation between session lifecycle state
and semantic turn state.
Responsibilities:
- Verify SessionState owns session and audio sequencing state.
- Verify TurnManager owns semantic turn transitions.
- Verify the expected lifecycle of a complete turn.
- Verify interruption and invalid transitions.
Non-responsibilities:
- Testing STT, Agent, or TTS providers.
- Testing WebSocket transport.
- Testing provider configuration.
"""
import pytest
from voice_assistant.audio_gateway.app.session import SessionState
from voice_assistant.turn_manager import TurnManager, TurnState
def test_session_creates_sequential_turn_ids():
session = SessionState()
first_turn = session.start_turn()
assert first_turn == 1
assert session.active_turn_id == 1
assert session.expected_audio_sequence == 0
session.end_turn(first_turn)
second_turn = session.start_turn()
assert second_turn == 2
assert session.active_turn_id == 2
assert session.expected_audio_sequence == 0
def test_session_owns_audio_sequence():
session = SessionState()
turn_id = session.start_turn()
session.accept_audio_chunk(
turn_id=turn_id,
sequence_number=0,
payload_size=320,
)
assert session.expected_audio_sequence == 1
session.accept_audio_chunk(
turn_id=turn_id,
sequence_number=1,
payload_size=320,
)
assert session.expected_audio_sequence == 2
def test_session_rejects_out_of_order_audio():
session = SessionState()
turn_id = session.start_turn()
with pytest.raises(
ValueError,
match="Expected sequence_number=0",
):
session.accept_audio_chunk(
turn_id=turn_id,
sequence_number=1,
payload_size=320,
)
def test_turn_manager_owns_semantic_turn_state():
manager = TurnManager()
turn = manager.start(turn_id=1)
assert turn.turn_id == 1
assert turn.state == TurnState.LISTENING
manager.on_partial(
turn_id=1,
text="Olá",
)
assert turn.state == TurnState.LISTENING
manager.on_final(
turn_id=1,
text="Olá!",
)
assert turn.state == TurnState.PROCESSING
manager.on_agent_response(turn_id=1)
assert turn.state == TurnState.SPEAKING
manager.on_tts_complete(turn_id=1)
assert turn.state == TurnState.COMPLETED
def test_turn_manager_rejects_invalid_transition():
manager = TurnManager()
manager.start(turn_id=1)
with pytest.raises(
RuntimeError,
match="Cannot start speaking",
):
manager.on_agent_response(turn_id=1)
def test_turn_manager_interrupts_active_turn():
manager = TurnManager()
manager.start(turn_id=1)
turn = manager.interrupt(turn_id=1)
assert turn.state == TurnState.INTERRUPTED
assert manager.active_turn is turn
def test_turn_manager_stop_requires_completed_state():
manager = TurnManager()
manager.start(turn_id=1)
manager.on_final(
turn_id=1,
text="Olá!",
)
with pytest.raises(
RuntimeError,
match="Cannot stop turn while turn is processing",
):
manager.stop(turn_id=1)
def test_turn_manager_stop_allows_completed_turn():
manager = TurnManager()
manager.start(turn_id=1)
manager.on_final(
turn_id=1,
text="Olá!",
)
manager.on_agent_response(turn_id=1)
manager.on_tts_complete(turn_id=1)
turn = manager.stop(turn_id=1)
assert turn.state == TurnState.COMPLETED
assert turn.final_text == "Olá!"
def test_session_and_turn_manager_complete_same_turn():
session = SessionState()
manager = TurnManager()
turn_id = session.start_turn()
turn = manager.start(turn_id)
assert session.active_turn_id == turn_id
assert turn.turn_id == turn_id
assert turn.state == TurnState.LISTENING
manager.on_final(
turn_id,
"Olá!",
)
manager.on_agent_response(turn_id)
manager.on_tts_complete(turn_id)
completed_turn = manager.stop(turn_id)
assert completed_turn.state == TurnState.COMPLETED
manager.clear(turn_id)
session.end_turn(turn_id)
assert manager.active_turn is None
assert session.active_turn_id is None
def test_session_and_turn_manager_must_use_same_turn_id():
session = SessionState()
manager = TurnManager()
session_turn_id = session.start_turn()
manager.start(1)
with pytest.raises(
ValueError,
match="Unexpected turn_id=2",
):
session.accept_audio_chunk(
turn_id=2,
sequence_number=0,
payload_size=320,
)
assert session.active_turn_id == session_turn_id
assert manager.active_turn is not None
assert manager.active_turn.turn_id == 1
def test_session_ends_turn_after_turn_manager_completion():
session = SessionState()
manager = TurnManager()
turn_id = session.start_turn()
manager.start(turn_id)
manager.on_final(
turn_id,
"Olá!",
)
manager.on_agent_response(turn_id)
manager.on_tts_complete(turn_id)
manager.stop(turn_id)
manager.clear(turn_id)
session.end_turn(turn_id)
assert session.active_turn_id is None
def test_session_and_turn_manager_interrupt_same_turn():
session = SessionState()
manager = TurnManager()
turn_id = session.start_turn()
manager.start(turn_id)
session.interrupt(turn_id)
interrupted_turn = manager.interrupt(turn_id)
assert session.active_turn_id is None
assert manager.active_turn is interrupted_turn
assert interrupted_turn.turn_id == turn_id
assert interrupted_turn.state == TurnState.INTERRUPTED
def test_session_can_start_next_turn_after_interrupt():
session = SessionState()
manager = TurnManager()
first_turn_id = session.start_turn()
manager.start(first_turn_id)
session.interrupt(first_turn_id)
manager.interrupt(first_turn_id)
manager.clear(first_turn_id)
second_turn_id = session.start_turn()
manager.start(second_turn_id)
assert second_turn_id == 2
assert session.active_turn_id == 2
assert manager.active_turn is not None
assert manager.active_turn.turn_id == 2
assert manager.active_turn.state == TurnState.LISTENING

307
tests/test_stt_abort.py Normal file
View File

@@ -0,0 +1,307 @@
"""
Tests for STT session abort behavior.
Purpose:
Verify that STT implementations support interruption
without delivering stale transcription results.
Responsibilities:
- Validate immediate abort.
- Verify pending transcript events are discarded.
- Verify repeated abort calls are safe.
- Verify audio is rejected after abort.
"""
import pytest
import asyncio
from unittest.mock import AsyncMock
from voice_assistant.stt.nvidia import NvidiaSTTAdapter
from voice_assistant.stt.models import TranscriptEvent
from voice_assistant.stt.mock import MockSTTAdapter
@pytest.mark.anyio
async def test_mock_stt_abort_discards_pending_events():
stt = MockSTTAdapter()
await stt.start_session()
await stt.emit_partial("Olá")
await stt.emit_final("Olá, tudo bem?")
await stt.abort_session()
events = [
event
async for event in stt.events()
]
assert events == []
@pytest.mark.anyio
async def test_mock_stt_abort_rejects_audio():
stt = MockSTTAdapter()
await stt.start_session()
await stt.abort_session()
with pytest.raises(
RuntimeError,
match="STT session has not been started",
):
await stt.send_audio(b"\x00\x01")
@pytest.mark.anyio
async def test_mock_stt_abort_is_idempotent():
stt = MockSTTAdapter()
await stt.start_session()
await stt.abort_session()
await stt.abort_session()
assert stt._started is False
@pytest.mark.anyio
async def test_mock_stt_abort_unblocks_event_consumer():
stt = MockSTTAdapter()
await stt.start_session()
async def consume():
return [
event
async for event in stt.events()
]
import asyncio
consumer = asyncio.create_task(consume())
# Allow the consumer to start waiting for events.
await asyncio.sleep(0)
await stt.abort_session()
events = await asyncio.wait_for(
consumer,
timeout=1.0,
)
assert events == []
@pytest.mark.anyio
async def test_nvidia_abort_without_started_session():
stt = NvidiaSTTAdapter()
await stt.abort_session()
await stt.abort_session()
assert stt._started is False
assert stt._websocket is None
assert stt._http is None
@pytest.mark.anyio
async def test_nvidia_abort_discards_pending_events():
stt = NvidiaSTTAdapter()
stt._started = True
await stt._events.put(
TranscriptEvent(
type="partial",
text="Transcrição antiga",
)
)
await stt.abort_session()
events = [
event
async for event in stt.events()
]
assert events == []
@pytest.mark.anyio
async def test_nvidia_abort_unblocks_consumer():
stt = NvidiaSTTAdapter()
stt._started = True
async def consume():
return [
event
async for event in stt.events()
]
consumer = asyncio.create_task(consume())
await asyncio.sleep(0)
await stt.abort_session()
events = await asyncio.wait_for(
consumer,
timeout=1.0,
)
assert events == []
@pytest.mark.anyio
async def test_nvidia_abort_closes_resources_without_done():
stt = NvidiaSTTAdapter()
websocket = AsyncMock()
http_client = AsyncMock()
stt._started = True
stt._websocket = websocket
stt._http = http_client
await stt.abort_session()
websocket.close.assert_awaited_once()
websocket.send.assert_not_awaited()
http_client.aclose.assert_awaited_once()
assert stt._websocket is None
assert stt._http is None
assert stt._started is False
@pytest.mark.anyio
async def test_nvidia_abort_cancels_receiver_task():
stt = NvidiaSTTAdapter()
stt._started = True
started = asyncio.Event()
finished = asyncio.Event()
async def receiver():
started.set()
try:
await asyncio.Event().wait()
finally:
finished.set()
task = asyncio.create_task(receiver())
stt._receiver_task = task
await started.wait()
await stt.abort_session()
assert task.done()
assert task.cancelled()
assert finished.is_set()
assert stt._receiver_task is None
@pytest.mark.anyio
async def test_nvidia_concurrent_abort():
"""
Verify that concurrent abort requests produce a single
end-of-stream marker and leave the adapter stopped.
"""
stt = NvidiaSTTAdapter()
stt._started = True
await asyncio.gather(
stt.abort_session(),
stt.abort_session(),
)
assert stt._started is False
assert stt._receiver_task is None
assert stt._websocket is None
assert stt._http is None
assert stt._events.qsize() == 1
@pytest.mark.anyio
async def test_nvidia_abort_does_not_deliver_late_events():
"""Queued transcripts must not survive an explicit abort."""
stt = NvidiaSTTAdapter()
stt._started = True
await stt._events.put(
TranscriptEvent(
type="partial",
text="Resultado antigo",
)
)
await stt.abort_session()
events = [
event
async for event in stt.events()
]
assert events == []
@pytest.mark.anyio
async def test_nvidia_concurrent_abort_closes_resources_once():
"""
Verify that concurrent abort requests do not close
provider resources more than once.
"""
stt = NvidiaSTTAdapter()
websocket = AsyncMock()
http_client = AsyncMock()
stt._started = True
stt._websocket = websocket
stt._http = http_client
# Force both abort operations to overlap during cleanup.
close_started = asyncio.Event()
allow_close = asyncio.Event()
async def slow_close():
close_started.set()
await allow_close.wait()
websocket.close.side_effect = slow_close
first_abort = asyncio.create_task(
stt.abort_session()
)
try:
await asyncio.wait_for(
close_started.wait(),
timeout=1.0,
)
second_abort = asyncio.create_task(
stt.abort_session()
)
# Give the second abort a chance to enter cleanup.
await asyncio.sleep(0)
finally:
allow_close.set()
await asyncio.gather(
first_abort,
second_abort,
)
websocket.close.assert_awaited_once()
http_client.aclose.assert_awaited_once()
assert stt._events.qsize() == 1

39
tests/test_tts.py Normal file
View File

@@ -0,0 +1,39 @@
import pytest
from voice_assistant.stt.mock import MockSTTAdapter
@pytest.mark.anyio
async def test_mock_stt_lifecycle():
stt = MockSTTAdapter()
await stt.start_session()
await stt.send_audio(b"\x00\x00" * 160)
await stt.emit_partial("eu gostaria de saber")
await stt.emit_final(
"eu gostaria de saber o horário de funcionamento"
)
await stt.stop_session()
events = [event async for event in stt.events()]
assert len(events) == 2
assert events[0].type == "partial"
assert events[0].text == "eu gostaria de saber"
assert events[1].type == "final"
assert events[1].text == (
"eu gostaria de saber o horário de funcionamento"
)
@pytest.mark.anyio
async def test_mock_stt_rejects_audio_before_start():
stt = MockSTTAdapter()
with pytest.raises(RuntimeError, match="has not been started"):
await stt.send_audio(b"\x00\x00")

32
tests/test_tts_adapter.py Normal file
View File

@@ -0,0 +1,32 @@
import pytest
from voice_assistant.tts import MockTTSAdapter
@pytest.mark.anyio
async def test_mock_tts_synthesizes_audio_chunks():
tts= MockTTSAdapter()
chunks = [
chunk
async for chunk in tts.synthesize("Olá, tudo bem?")
]
assert chunks == [
b"\x01\x02\x03\x04",
b"\x05\x06\x07\x08",
]
@pytest.mark.anyio
async def test_mock_tts_records_text():
tts = MockTTSAdapter()
chunks = [
chunk
async for chunk in tts.synthesize("Olá")
]
assert chunks
assert tts.calls == ["Olá"], "A mensagem de retorno deve ser Olá"

194
tests/test_turn_manager.py Normal file
View File

@@ -0,0 +1,194 @@
import pytest
from voice_assistant.turn_manager import (
TurnManager,
TurnState,
)
def test_start_turn():
manager = TurnManager()
turn = manager.start(1)
assert turn.turn_id == 1
assert turn.state == TurnState.LISTENING
assert turn.partial_text == ""
assert turn.final_text is None
def test_partial_replaces_previous_partial():
manager = TurnManager()
manager.start(1)
manager.on_partial(1, "Ô Alex")
assert manager.active_turn is not None
assert manager.active_turn.partial_text == "Ô Alex"
manager.on_partial(1, "Ô Alex tudo bem")
assert manager.active_turn.partial_text == "Ô Alex tudo bem"
def test_final_moves_turn_to_processing():
manager = TurnManager()
manager.start(1)
manager.on_partial(1, "Ô Alex tudo bem")
manager.on_final(1, "Ô Alex tudo bem?")
assert manager.active_turn is not None
assert manager.active_turn.final_text == "Ô Alex tudo bem?"
assert manager.active_turn.partial_text == "Ô Alex tudo bem?"
assert manager.active_turn.state == TurnState.PROCESSING
def test_cannot_start_second_turn():
manager = TurnManager()
manager.start(1)
with pytest.raises(
RuntimeError,
match="A turn is already active",
):
manager.start(2)
def test_cannot_stop_before_completed():
manager = TurnManager()
manager.start(1)
with pytest.raises(
RuntimeError,
match="final transcript",
):
manager.stop(1)
def test_interrupt():
manager = TurnManager()
manager.start(1)
manager.interrupt(1)
assert manager.active_turn is not None
assert (
manager.active_turn.state
== TurnState.INTERRUPTED
)
def test_clear_turn():
manager = TurnManager()
manager.start(1)
manager.clear(1)
assert manager.active_turn is None
def test_agent_response_moves_turn_to_speaking():
manager = TurnManager()
manager.start(1)
manager.on_final(
1,
"Qual o horário?",
)
manager.on_agent_response(1)
assert manager.active_turn is not None
assert (
manager.active_turn.state
== TurnState.SPEAKING
)
def test_tts_completion_moves_turn_to_completed():
manager = TurnManager()
manager.start(1)
manager.on_final(
1,
"Qual o horário?",
)
manager.on_agent_response(1)
manager.on_tts_complete(1)
assert manager.active_turn is not None
assert (
manager.active_turn.state
== TurnState.COMPLETED
)
def test_agent_response_requires_processing_state():
manager = TurnManager()
manager.start(1)
with pytest.raises(
RuntimeError,
match="turn is listening",
):
manager.on_agent_response(1)
def test_tts_completion_requires_speaking_state():
manager = TurnManager()
manager.start(1)
with pytest.raises(
RuntimeError,
match="turn is listening",
):
manager.on_tts_complete(1)
def test_cannot_stop_before_completed():
manager = TurnManager()
manager.start(1)
with pytest.raises(
RuntimeError,
match="Cannot stop turn while turn is listening",
):
manager.stop(1)
def test_clear_removes_completed_turn():
manager = TurnManager()
manager.start(1)
manager.on_final(
1,
"Olá!",
)
manager.on_agent_response(1)
manager.on_tts_complete(1)
manager.clear(1)
assert manager.active_turn is None
def test_clear_rejects_unexpected_turn_id():
manager = TurnManager()
manager.start(1)
with pytest.raises(
ValueError,
match="Unexpected turn_id=2",
):
manager.clear(2)

View File

@@ -0,0 +1,820 @@
"""
Tests for the Voice Pipeline lifecycle orchestration.
Purpose:
Verify that the VoicePipeline coordinates SessionState and
TurnManager without coupling either component to transport details.
Responsibilities:
- Validate turn creation.
- Validate semantic turn completion.
- Validate coordinated interruption.
- Verify SessionState and TurnManager remain independently modeled.
Non-responsibilities:
- Testing WebSocket transport.
- Testing STT, Agent, or TTS providers.
"""
import pytest
import asyncio
from voice_assistant.audio_gateway.app.session import SessionState
from voice_assistant.turn_manager import TurnManager, TurnState
from voice_assistant.voice_pipeline import VoicePipeline
from voice_assistant.agent.mock import MockAgentAdapter
from voice_assistant.tts.mock import MockTTSAdapter
def create_pipeline() -> tuple[
VoicePipeline,
SessionState,
TurnManager,
MockAgentAdapter,
MockTTSAdapter,
]:
session = SessionState()
turn_manager = TurnManager()
agent = MockAgentAdapter()
tts = MockTTSAdapter()
pipeline = VoicePipeline(
session=session,
turn_manager=turn_manager,
agent=agent,
tts=tts,
)
return (
pipeline,
session,
turn_manager,
agent,
tts,
)
def test_pipeline_starts_turn():
pipeline, session, turn_manager, _, _ = create_pipeline()
turn = pipeline.start_turn()
assert turn.turn_id == 1
assert turn.state == TurnState.LISTENING
assert session.active_turn_id == 1
assert turn_manager.active_turn is turn
def test_pipeline_completes_turn():
pipeline, session, turn_manager, _, _ = create_pipeline()
turn = pipeline.start_turn()
turn_manager.on_final(
turn.turn_id,
"Olá!",
)
turn_manager.on_agent_response(turn.turn_id)
turn_manager.on_tts_complete(turn.turn_id)
completed_turn = pipeline.complete_turn(
turn.turn_id,
)
assert completed_turn.state == TurnState.COMPLETED
assert session.active_turn_id is None
assert turn_manager.active_turn is None
def test_pipeline_rejects_incomplete_turn():
pipeline, session, turn_manager, _, _ = create_pipeline()
turn = pipeline.start_turn()
with pytest.raises(
RuntimeError,
match="Cannot stop turn while turn is listening",
):
pipeline.complete_turn(turn.turn_id)
# Failed completion must not destroy the active turn.
assert session.active_turn_id == turn.turn_id
assert turn_manager.active_turn is turn
def test_pipeline_interrupts_turn():
pipeline, session, turn_manager, _, _ = create_pipeline()
turn = pipeline.start_turn()
interrupted_turn = pipeline.interrupt_turn(
turn.turn_id,
)
assert interrupted_turn.state == TurnState.INTERRUPTED
assert session.active_turn_id is None
assert turn_manager.active_turn is None
def test_pipeline_can_start_turn_after_interrupt():
pipeline, session, turn_manager, _, _ = create_pipeline()
first_turn = pipeline.start_turn()
pipeline.interrupt_turn(
first_turn.turn_id,
)
second_turn = pipeline.start_turn()
assert second_turn.turn_id == 2
assert second_turn.state == TurnState.LISTENING
assert session.active_turn_id == 2
assert turn_manager.active_turn is second_turn
@pytest.mark.anyio
async def test_pipeline_processes_transcript_through_agent():
(
pipeline,
session,
turn_manager,
agent,
_,
) = create_pipeline()
turn = pipeline.start_turn()
turn_manager.on_final(
turn.turn_id,
"Olá!",
)
result = await pipeline.process_transcript(
session_id=session.session_id,
turn_id=turn.turn_id,
text="Olá!",
)
assert result.response_text == (
"Resposta simulada para: Olá!"
)
assert agent.calls == [
(
session.session_id,
turn.turn_id,
"Olá!",
)
]
assert turn_manager.active_turn is not None
assert (
turn_manager.active_turn.state
== TurnState.SPEAKING
)
@pytest.mark.anyio
async def test_pipeline_streams_tts_audio():
(
pipeline,
session,
turn_manager,
_,
tts,
) = create_pipeline()
turn = pipeline.start_turn()
turn_manager.on_final(
turn.turn_id,
"Olá!",
)
result = await pipeline.process_transcript(
session_id=session.session_id,
turn_id=turn.turn_id,
text="Olá!",
)
chunks = [
chunk
async for chunk in pipeline.synthesize_response(
result.response_text,
)
]
assert chunks == [
b"\x01\x02\x03\x04",
b"\x05\x06\x07\x08",
]
assert tts.calls == [
"Resposta simulada para: Olá!",
]
@pytest.mark.anyio
async def test_pipeline_processes_complete_voice_turn():
(
pipeline,
session,
turn_manager,
agent,
tts,
) = create_pipeline()
turn = pipeline.start_turn()
turn_manager.on_final(
turn.turn_id,
"Olá!",
)
result = await pipeline.process_transcript(
session_id=session.session_id,
turn_id=turn.turn_id,
text="Olá!",
)
audio_chunks = [
chunk
async for chunk in pipeline.synthesize_response(
result.response_text,
)
]
turn_manager.on_tts_complete(
turn.turn_id,
)
assert result.response_text == (
"Resposta simulada para: Olá!"
)
assert audio_chunks == [
b"\x01\x02\x03\x04",
b"\x05\x06\x07\x08",
]
assert turn_manager.active_turn is not None
assert (
turn_manager.active_turn.state
== TurnState.COMPLETED
)
assert agent.calls == [
(
session.session_id,
turn.turn_id,
"Olá!",
)
]
assert tts.calls == [
"Resposta simulada para: Olá!",
]
def test_pipeline_handles_partial_transcript():
pipeline, _, _, _, _ = create_pipeline()
turn = pipeline.start_turn()
pipeline.on_partial(
turn.turn_id,
"Olá",
)
assert turn.partial_text == "Olá"
assert turn.state == TurnState.LISTENING
def test_pipeline_handles_final_transcript():
pipeline, _, _, _, _ = create_pipeline()
turn = pipeline.start_turn()
pipeline.on_final(
turn.turn_id,
"Olá!",
)
assert turn.final_text == "Olá!"
assert turn.partial_text == "Olá!"
assert turn.state == TurnState.PROCESSING
def test_pipeline_handles_tts_completion():
pipeline, _, _, _, _ = create_pipeline()
turn = pipeline.start_turn()
pipeline.on_final(
turn.turn_id,
"Olá!",
)
# The Agent transition is part of the normal semantic lifecycle.
pipeline.on_agent_response(
turn.turn_id,
)
pipeline.on_tts_complete(
turn.turn_id,
)
assert turn.state == TurnState.COMPLETED
def test_pipeline_closes_session_without_active_turn():
pipeline, session, turn_manager, _, _ = create_pipeline()
pipeline.close_session()
assert session.closed is True
assert session.active_turn_id is None
assert turn_manager.active_turn is None
def test_pipeline_closes_session_with_active_turn():
pipeline, session, turn_manager, _, _ = create_pipeline()
turn = pipeline.start_turn()
pipeline.close_session()
assert session.closed is True
assert session.active_turn_id is None
assert turn_manager.active_turn is None
assert turn.state == TurnState.INTERRUPTED
def test_pipeline_cannot_start_turn_after_session_close():
pipeline, session, turn_manager, _, _ = create_pipeline()
pipeline.close_session()
with pytest.raises(
RuntimeError,
match="Session is closed",
):
pipeline.start_turn()
def test_pipeline_interrupt_if_active():
pipeline, session, turn_manager, _, _ = create_pipeline()
turn = pipeline.start_turn()
interrupted = pipeline.interrupt_if_active(
turn.turn_id,
)
assert interrupted is True
assert session.active_turn_id is None
assert turn_manager.active_turn is None
assert turn.state == TurnState.INTERRUPTED
def test_pipeline_interrupt_if_active_unknown_turn():
pipeline, session, turn_manager, _, _ = create_pipeline()
turn = pipeline.start_turn()
interrupted = pipeline.interrupt_if_active(
turn.turn_id + 1,
)
assert interrupted is False
assert session.active_turn_id == turn.turn_id
assert turn_manager.active_turn is turn
assert turn.state == TurnState.LISTENING
@pytest.mark.anyio
async def test_pipeline_registers_turn_task():
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
async def worker():
await asyncio.sleep(0)
task = asyncio.create_task(worker())
pipeline.register_turn_task(task)
assert pipeline.turn_task is task
await task
@pytest.mark.anyio
async def test_pipeline_rejects_duplicate_running_task():
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
started = asyncio.Event()
release = asyncio.Event()
async def worker():
started.set()
await release.wait()
task = asyncio.create_task(worker())
try:
await started.wait()
pipeline.register_turn_task(task)
with pytest.raises(
RuntimeError,
match="A turn execution task is already active",
):
pipeline.register_turn_task(task)
finally:
release.set()
await task
@pytest.mark.anyio
async def test_pipeline_rejects_task_without_active_turn():
pipeline, _, _, _, _ = create_pipeline()
async def worker():
await asyncio.sleep(0)
task = asyncio.create_task(worker())
try:
with pytest.raises(
RuntimeError,
match="Cannot register task without an active turn",
):
pipeline.register_turn_task(task)
finally:
await task
@pytest.mark.anyio
async def test_pipeline_cancels_running_turn_task():
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
started = asyncio.Event()
cancelled = asyncio.Event()
async def worker():
started.set()
try:
await asyncio.Event().wait()
finally:
cancelled.set()
task = asyncio.create_task(worker())
await started.wait()
pipeline.register_turn_task(task)
await pipeline.cancel_turn_task()
assert cancelled.is_set()
assert task.done()
assert task.cancelled()
assert pipeline.turn_task is None
@pytest.mark.anyio
async def test_pipeline_cancel_without_registered_task():
pipeline, _, _, _, _ = create_pipeline()
await pipeline.cancel_turn_task()
assert pipeline.turn_task is None
@pytest.mark.anyio
async def test_pipeline_cancel_completed_task():
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
async def worker():
return None
task = asyncio.create_task(worker())
pipeline.register_turn_task(task)
await task
await pipeline.cancel_turn_task()
assert task.done()
assert pipeline.turn_task is None
@pytest.mark.anyio
async def test_pipeline_rejects_self_cancellation():
"""
Verify that a turn task cannot cancel itself.
The task reference must remain available so another
task can subsequently perform cleanup.
"""
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
async def worker():
await pipeline.cancel_turn_task()
task = asyncio.create_task(worker())
pipeline.register_turn_task(task)
# Self-cancellation must be rejected.
with pytest.raises(
RuntimeError,
match="A turn task cannot cancel itself",
):
await task
# The failed task remains registered until explicit cleanup.
assert pipeline.turn_task is task
# Cleanup is performed from the test task, not the worker.
with pytest.raises(
RuntimeError,
match="A turn task cannot cancel itself",
):
await pipeline.cancel_turn_task()
# The reference must be released after the completed task
# has been awaited, even when its exception is propagated.
assert pipeline.turn_task is None
@pytest.mark.anyio
async def test_pipeline_propagates_turn_task_failure():
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
async def worker():
raise ValueError(
"Provider execution failed."
)
task = asyncio.create_task(worker())
pipeline.register_turn_task(task)
# Allow the worker to execute and finish with its error.
await asyncio.sleep(0)
assert task.done()
with pytest.raises(
ValueError,
match="Provider execution failed",
):
await pipeline.cancel_turn_task()
assert task.done()
assert pipeline.turn_task is None
@pytest.mark.anyio
async def test_pipeline_propagates_waiter_cancellation():
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
worker_started = asyncio.Event()
allow_cleanup = asyncio.Event()
async def worker():
worker_started.set()
try:
await asyncio.Event().wait()
finally:
await allow_cleanup.wait()
task = asyncio.create_task(worker())
await worker_started.wait()
pipeline.register_turn_task(task)
waiter = asyncio.create_task(
pipeline.cancel_turn_task()
)
try:
# Allow cancellation to reach the worker.
await asyncio.sleep(0)
waiter.cancel()
with pytest.raises(asyncio.CancelledError):
await waiter
# The worker is still completing its cancellation cleanup.
assert pipeline.turn_task is task
finally:
allow_cleanup.set()
await pipeline.cancel_turn_task()
assert task.done()
assert pipeline.turn_task is None
def test_pipeline_identifies_active_turn():
pipeline, _, _, _, _ = create_pipeline()
turn = pipeline.start_turn()
assert pipeline.is_active_turn(turn.turn_id) is True
def test_pipeline_rejects_inactive_turn_id():
pipeline, _, _, _, _ = create_pipeline()
turn = pipeline.start_turn()
assert pipeline.is_active_turn(turn.turn_id + 1) is False
pipeline.interrupt_turn(turn.turn_id)
assert pipeline.is_active_turn(turn.turn_id) is False
@pytest.mark.anyio
async def test_pipeline_waits_for_completed_turn_task():
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
completed = asyncio.Event()
async def worker():
completed.set()
task = asyncio.create_task(worker())
pipeline.register_turn_task(task)
await pipeline.wait_turn_task()
assert completed.is_set()
assert task.done()
assert pipeline.turn_task is None
@pytest.mark.anyio
async def test_pipeline_wait_propagates_task_failure():
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
async def worker():
raise ValueError("Agent processing failed.")
task = asyncio.create_task(worker())
pipeline.register_turn_task(task)
with pytest.raises(
ValueError,
match="Agent processing failed",
):
await pipeline.wait_turn_task()
assert pipeline.turn_task is None
@pytest.mark.anyio
async def test_pipeline_wait_without_registered_task():
pipeline, _, _, _, _ = create_pipeline()
await pipeline.wait_turn_task()
assert pipeline.turn_task is None
@pytest.mark.anyio
async def test_pipeline_cancel_rejects_invalid_timeout():
pipeline, _, _, _, _ = create_pipeline()
with pytest.raises(
ValueError,
match="Turn cancellation timeout must be positive",
):
await pipeline.cancel_turn_task(timeout=0)
@pytest.mark.anyio
async def test_pipeline_cancel_completes_within_timeout():
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
started = asyncio.Event()
async def worker():
started.set()
await asyncio.Event().wait()
task = asyncio.create_task(worker())
await started.wait()
pipeline.register_turn_task(task)
await pipeline.cancel_turn_task(timeout=1.0)
assert task.done()
assert task.cancelled()
assert pipeline.turn_task is None
@pytest.mark.anyio
async def test_pipeline_cancel_timeout_preserves_task_ownership():
pipeline, _, _, _, _ = create_pipeline()
pipeline.start_turn()
started = asyncio.Event()
allow_cleanup = asyncio.Event()
cancellation_received = asyncio.Event()
async def worker():
started.set()
try:
await asyncio.Event().wait()
finally:
cancellation_received.set()
await allow_cleanup.wait()
task = asyncio.create_task(worker())
await started.wait()
pipeline.register_turn_task(task)
try:
with pytest.raises(
TimeoutError,
match="Turn task cancellation timed out",
):
await pipeline.cancel_turn_task(timeout=0.01)
assert cancellation_received.is_set()
assert not task.done()
assert pipeline.turn_task is task
finally:
allow_cleanup.set()
await pipeline.cancel_turn_task(timeout=1.0)
assert task.done()
assert pipeline.turn_task is None
@pytest.mark.anyio
async def test_pipeline_rejects_new_turn_while_previous_task_runs():
pipeline, session, _, _, _ = create_pipeline()
first_turn = pipeline.start_turn()
started = asyncio.Event()
release = asyncio.Event()
async def worker():
started.set()
await release.wait()
task = asyncio.create_task(worker())
await started.wait()
pipeline.register_turn_task(task)
# Simulate semantic interruption without task cancellation.
pipeline.interrupt_turn(first_turn.turn_id)
try:
with pytest.raises(
RuntimeError,
match="previous turn task is still running",
):
pipeline.start_turn()
assert session.active_turn_id is None
finally:
release.set()
await pipeline.wait_turn_task()
second_turn = pipeline.start_turn()
assert second_turn.turn_id == 2
assert session.active_turn_id == 2

147
tests/test_voxcpm_tts.py Normal file
View File

@@ -0,0 +1,147 @@
import httpx
import pytest
from voice_assistant.tts.voxcpm import VoxCPMTTSAdapter
class MockHTTPStream:
def __init__(
self,
chunks: list[bytes],
status_code: int = 200,
) -> None:
self.chunks = chunks
self.status_code = status_code
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
def raise_for_status(self):
if self.status_code >= 400:
raise httpx.HTTPStatusError(
"mock error",
request=httpx.Request("POST", "http://test"),
response=httpx.Response(
status_code=self.status_code,
),
)
async def aiter_bytes(self):
for chunk in self.chunks:
yield chunk
class MockHTTPClient:
last_request = None
def __init__(self, *args, **kwargs) -> None:
pass
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
def stream(self, method, url, json):
self.last_request = {
"method": method,
"url": url,
"json": json,
}
MockHTTPClient.last_request = self.last_request
return MockHTTPStream(
[
b"\x01\x02\x03\x04",
b"\x05\x06\x07\x08",
]
)
@pytest.mark.anyio
async def test_voxcpm_tts_streams_audio(monkeypatch):
monkeypatch.setattr(
httpx,
"AsyncClient",
MockHTTPClient,
)
tts = VoxCPMTTSAdapter(
server="http://127.0.0.1:8000",
model="openbmb/VoxCPM2",
voice="default",
)
chunks = [
chunk
async for chunk in tts.synthesize(
"Olá Alex, tudo bem?"
)
]
assert chunks == [
b"\x01\x02\x03\x04",
b"\x05\x06\x07\x08",
]
@pytest.mark.anyio
async def test_voxcpm_tts_sends_expected_payload(monkeypatch):
monkeypatch.setattr(
httpx,
"AsyncClient",
MockHTTPClient,
)
tts = VoxCPMTTSAdapter(
server="http://127.0.0.1:8000",
model="openbmb/VoxCPM2",
voice="default",
)
chunks = [
chunk
async for chunk in tts.synthesize("Olá")
]
assert chunks
request = MockHTTPClient.last_request
assert request is not None
assert request["method"] == "POST"
assert request["url"] == (
"http://127.0.0.1:8000/v1/audio/speech"
)
assert request["json"] == {
"input": "Olá",
"model": "openbmb/VoxCPM2",
"voice": "default",
"response_format": "pcm",
"stream": True,
"stream_format": "audio",
}
@pytest.mark.anyio
async def test_voxcpm_tts_rejects_empty_text():
tts = VoxCPMTTSAdapter(
server="http://127.0.0.1:8000",
model="openbmb/VoxCPM2",
voice="default",
)
with pytest.raises(
ValueError,
match="Text cannot be empty",
):
chunks = [
chunk
async for chunk in tts.synthesize("")
]

View File

@@ -0,0 +1,29 @@
import pytest
from voice_assistant.tts.voxcpm import VoxCPMTTSAdapter
@pytest.mark.anyio
async def test_voxcpm_tts_real():
tts = VoxCPMTTSAdapter(
server="http://127.0.0.1:8000",
model="openbmb/VoxCPM2",
voice="default",
)
chunks = [
chunk
async for chunk in tts.synthesize(
"Olá Alex, este é um teste do VoxCPM."
)
]
assert chunks
assert all(isinstance(chunk, bytes) for chunk in chunks)
total_bytes = sum(len(chunk) for chunk in chunks)
assert total_bytes > 0
print(f"Chunks recebidos : {len(chunks)}")
print(f"Bytes recebidos : {total_bytes}")

View File

@@ -0,0 +1,246 @@
import asyncio
import json
import wave
import websockets
GATEWAY_URL = "ws://127.0.0.1:8000/ws/audio"
AUDIO_FILE = (
"/home/ubuntu/voice-assistant/audios/chafik_normalize.wav"
)
CHUNK_FRAMES = 1600
async def main() -> None:
print("========================================")
print("STT GATEWAY SMOKE TEST")
print("========================================")
print(f"Gateway : {GATEWAY_URL}")
print(f"Audio : {AUDIO_FILE}")
print(f"Chunk : {CHUNK_FRAMES} frames")
print()
async with websockets.connect(GATEWAY_URL) as websocket:
# -------------------------------------------------------------
# 1. session.start
# -------------------------------------------------------------
await websocket.send(
json.dumps(
{
"type": "session.start",
"audio": {
"encoding": "pcm_s16le",
"sample_rate": 16000,
"channels": 1,
},
}
)
)
response = json.loads(
await websocket.recv()
)
print(
f"[RECV] {response['type']}"
)
if response["type"] != "session.ready":
raise RuntimeError(
f"Expected session.ready, got {response}"
)
session_id = response["session_id"]
# -------------------------------------------------------------
# 2. audio.start
# -------------------------------------------------------------
await websocket.send(
json.dumps(
{
"type": "audio.start",
}
)
)
response = json.loads(
await websocket.recv()
)
print(
f"[RECV] {response['type']}"
)
if response["type"] != "turn.started":
raise RuntimeError(
f"Expected turn.started, got {response}"
)
turn_id = response["turn_id"]
print(
f"Session : {session_id}"
)
print(
f"Turn : {turn_id}"
)
print()
# -------------------------------------------------------------
# 3. Start a receiver task.
#
# This is necessary because partial/final events can arrive
# while we are still sending audio.
# -------------------------------------------------------------
received_events: list[dict] = []
async def receive_events() -> None:
while True:
message = await websocket.recv()
event = json.loads(message)
received_events.append(event)
event_type = event.get("type")
if event_type == "stt.partial":
print(
f"[PARTIAL] {event.get('text', '')}"
)
elif event_type == "stt.final":
print(
f"[FINAL ] {event.get('text', '')}"
)
elif event_type == "turn.ended":
print(
"[RECV] turn.ended"
)
break
elif event_type == "error":
print(
"[ERROR]",
event,
)
break
else:
print(
f"[RECV] {event_type}"
)
receiver_task = asyncio.create_task(
receive_events()
)
# -------------------------------------------------------------
# 4. Stream WAV as:
#
# audio.chunk JSON
# binary PCM
#
# The sequence number belongs to the logical chunk.
# -------------------------------------------------------------
with wave.open(AUDIO_FILE, "rb") as wav:
sequence_number = 0
while True:
audio = wav.readframes(CHUNK_FRAMES)
if not audio:
break
await websocket.send(
json.dumps(
{
"type": "audio.chunk",
"turn_id": turn_id,
"sequence_number": sequence_number,
}
)
)
await websocket.send(audio)
sequence_number += 1
print()
print(
f"Audio sent: {sequence_number} chunks"
)
# -------------------------------------------------------------
# 5. audio.stop
# -------------------------------------------------------------
await websocket.send(
json.dumps(
{
"type": "audio.stop",
"turn_id": turn_id,
}
)
)
# -------------------------------------------------------------
# 6. Wait for:
#
# stt.final
# turn.ended
#
# The receiver task handles both.
# -------------------------------------------------------------
await receiver_task
print()
print("========================================")
print("RESULT")
print("========================================")
partials = [
event
for event in received_events
if event.get("type") == "stt.partial"
]
finals = [
event
for event in received_events
if event.get("type") == "stt.final"
]
turn_ended = [
event
for event in received_events
if event.get("type") == "turn.ended"
]
print(
f"Partial events : {len(partials)}"
)
print(
f"Final events : {len(finals)}"
)
print(
f"Turn ended : {len(turn_ended)}"
)
if finals:
print()
print("Final transcript:")
print(
finals[-1].get("text", "")
)
print()
print("STT Gateway smoke test finished.")
if __name__ == "__main__":
asyncio.run(main())

114
tools/stt_smoke_test.py Normal file
View File

@@ -0,0 +1,114 @@
import argparse
import asyncio
from pathlib import Path
from riva.client.asr import AudioChunkFileIterator
from riva.client.realtime import RealtimeClientASR
NIM_SERVER = "127.0.0.1:9000"
INPUT_FILE = "/home/ubuntu/voice-assistant/audios/chafik_normalize.wav"
CHUNK_FRAMES = 1600
def build_args() -> argparse.Namespace:
return argparse.Namespace(
input_file=INPUT_FILE,
mic=False,
list_devices=False,
duration=None,
input_device=None,
sample_rate_hz=16000,
num_channels=1,
file_streaming_chunk=CHUNK_FRAMES,
output_text=None,
prompt="",
server=NIM_SERVER,
ssl_root_cert=None,
ssl_client_cert=None,
ssl_client_key=None,
use_ssl=False,
metadata=None,
options=None,
max_message_length=67108864,
word_time_offsets=False,
max_alternatives=1,
profanity_filter=False,
automatic_punctuation=True,
no_verbatim_transcripts=False,
language_code="pt-BR",
model_name="",
boosted_lm_words=None,
boosted_lm_score=4.0,
speaker_diarization=False,
diarization_max_speakers=3,
start_history=-1,
start_threshold=-1.0,
stop_history=-1,
stop_threshold=-1.0,
stop_history_eou=-1,
stop_threshold_eou=-1.0,
custom_configuration="",
endpoint="/v1/realtime",
query_params="intent=transcription",
mint_api_key=None,
)
async def main() -> None:
input_path = Path(INPUT_FILE)
if not input_path.exists():
raise FileNotFoundError(
f"Audio file not found: {input_path}"
)
args = build_args()
print(f"NIM server : {args.server}")
print(f"Audio : {input_path}")
print(f"Language : {args.language_code}")
print(f"Chunk : {CHUNK_FRAMES} frames")
client = RealtimeClientASR(args=args)
audio_iterator = AudioChunkFileIterator(
input_file=str(input_path),
chunk_n_frames=CHUNK_FRAMES,
delay_callback=None,
)
try:
print("Connecting to NIM...")
await client.connect()
print("Connected.")
send_task = asyncio.create_task(
client.send_audio_chunks(audio_iterator)
)
receive_task = asyncio.create_task(
client.receive_responses()
)
await asyncio.gather(
send_task,
receive_task,
)
finally:
await client.disconnect()
print("Disconnected.")
if __name__ == "__main__":
asyncio.run(main())

View File

View File

@@ -0,0 +1,327 @@
import asyncio
import json
import wave
from pathlib import Path
import websockets
GATEWAY_URL = "ws://127.0.0.1:8001/ws/audio"
AUDIO_FILE = Path(
"/home/ubuntu/voice-assistant/audios/chafik_normalize.wav"
)
OUTPUT_FILE = Path(
"/tmp/voice_pipeline_response.pcm"
)
CHUNK_FRAMES = 1600
async def receive_messages(
websocket,
state: dict,
) -> None:
while True:
message = await websocket.recv()
if isinstance(message, bytes):
state["tts_chunks"] += 1
state["tts_bytes"] += len(message)
state["tts_audio"].extend(message)
print(
f"[TTS AUDIO] "
f"chunk={state['tts_chunks']} "
f"bytes={len(message)}"
)
continue
event = json.loads(message)
event_type = event.get("type")
if event_type == "stt.partial":
state["partial_count"] += 1
print(f"[PARTIAL] {event.get('text', '')}")
elif event_type == "stt.final":
state["final_text"] = event.get("text")
print(f"[FINAL ] {state['final_text']}")
elif event_type == "agent.response":
state["agent_response"] = event.get("text")
print(
f"[AGENT ] "
f"{state['agent_response']}"
)
elif event_type == "turn.ended":
state["turn_ended"] = True
print("[RECV] turn.ended")
return
elif event_type == "error":
raise RuntimeError(
f"Gateway error: {event}"
)
else:
print(f"[RECV] {event_type}")
async def send_audio(
websocket,
turn_id: int,
) -> int:
sent_chunks = 0
with wave.open(str(AUDIO_FILE), "rb") as audio:
sample_rate = audio.getframerate()
channels = audio.getnchannels()
sample_width = audio.getsampwidth()
if sample_rate != 16000:
raise RuntimeError(
f"Expected 16 kHz audio, got {sample_rate} Hz."
)
if channels != 1:
raise RuntimeError(
f"Expected mono audio, got {channels} channels."
)
if sample_width != 2:
raise RuntimeError(
f"Expected 16-bit PCM audio, got "
f"{sample_width * 8}-bit."
)
while True:
pcm = audio.readframes(CHUNK_FRAMES)
if not pcm:
break
await websocket.send(
json.dumps(
{
"type": "audio.chunk",
"turn_id": turn_id,
"sequence_number": sent_chunks,
}
)
)
await websocket.send(pcm)
sent_chunks += 1
return sent_chunks
async def main() -> None:
print("=" * 40)
print("VOICE PIPELINE SMOKE TEST")
print("=" * 40)
print(f"Gateway : {GATEWAY_URL}")
print(f"Audio : {AUDIO_FILE}")
print(f"Chunk : {CHUNK_FRAMES} frames")
print()
state = {
"partial_count": 0,
"final_text": None,
"agent_response": None,
"tts_chunks": 0,
"tts_bytes": 0,
"tts_audio": bytearray(),
"turn_ended": False,
}
async with websockets.connect(GATEWAY_URL) as websocket:
# ---------------------------------------------------------
# session.start
# ---------------------------------------------------------
await websocket.send(
json.dumps(
{
"type": "session.start",
"audio": {
"encoding": "pcm_s16le",
"sample_rate": 16000,
"channels": 1,
},
}
)
)
message = await websocket.recv()
if isinstance(message, bytes):
raise RuntimeError(
"Expected JSON session.ready."
)
session_ready = json.loads(message)
print(
f"[RECV] "
f"{session_ready.get('type')}"
)
if session_ready.get("type") != "session.ready":
raise RuntimeError(
f"Expected session.ready, got "
f"{session_ready}"
)
session_id = session_ready["session_id"]
# ---------------------------------------------------------
# audio.start
# ---------------------------------------------------------
await websocket.send(
json.dumps(
{
"type": "audio.start",
}
)
)
message = await websocket.recv()
if isinstance(message, bytes):
raise RuntimeError(
"Expected JSON turn.started."
)
turn_started = json.loads(message)
print(
f"[RECV] "
f"{turn_started.get('type')}"
)
if turn_started.get("type") != "turn.started":
raise RuntimeError(
f"Expected turn.started, got "
f"{turn_started}"
)
turn_id = turn_started["turn_id"]
print()
print(f"Session : {session_id}")
print(f"Turn : {turn_id}")
print()
# ---------------------------------------------------------
# Run sender and receiver concurrently
# ---------------------------------------------------------
receiver_task = asyncio.create_task(
receive_messages(
websocket,
state,
)
)
try:
sent_chunks = await send_audio(
websocket,
turn_id,
)
except Exception:
receiver_task.cancel()
raise
print()
print(
f"Audio sent: {sent_chunks} chunks"
)
# ---------------------------------------------------------
# Tell Gateway that user audio is complete.
# ---------------------------------------------------------
await websocket.send(
json.dumps(
{
"type": "audio.stop",
"turn_id": turn_id,
}
)
)
# ---------------------------------------------------------
# Wait for the entire pipeline to finish.
# ---------------------------------------------------------
await asyncio.wait_for(
receiver_task,
timeout=120.0,
)
# ---------------------------------------------------------
# Validate the complete STT -> Agent -> TTS pipeline.
# ---------------------------------------------------------
if not state["turn_ended"]:
raise RuntimeError("Turn did not complete.")
if not state["final_text"]:
raise RuntimeError("STT final transcript was not received.")
if not state["agent_response"]:
raise RuntimeError("Agent response was not received.")
if state["tts_chunks"] == 0:
raise RuntimeError("No TTS audio was received.")
if state["tts_bytes"] == 0:
raise RuntimeError("TTS output is empty.")
# -------------------------------------------------------------
# Save TTS audio
# -------------------------------------------------------------
OUTPUT_FILE.write_bytes(
bytes(state["tts_audio"])
)
print()
print("=" * 40)
print("RESULT")
print("=" * 40)
print(f"Session : {session_id}")
print(f"Turn : {turn_id}")
print(
f"Audio sent : {sent_chunks}"
)
print(
f"Partial events : {state['partial_count']}"
)
print(
f"Final transcript: {state['final_text']}"
)
print(
f"Agent response : {state['agent_response']}"
)
print(
f"TTS chunks : {state['tts_chunks']}"
)
print(
f"TTS bytes : {state['tts_bytes']}"
)
print(
f"TTS output : {OUTPUT_FILE}"
)
print(
f"Turn ended : {state['turn_ended']}"
)
print()
print(
"Voice pipeline smoke test finished."
)
if __name__ == "__main__":
asyncio.run(main())

1362
uv.lock generated Normal file

File diff suppressed because it is too large Load Diff