feat: establish voice assistant V1 baseline
This commit is contained in:
36
.gitignore
vendored
Normal file
36
.gitignore
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.egg-info/
|
||||
build/
|
||||
dist/
|
||||
wheels/
|
||||
|
||||
# Virtual environments
|
||||
.venv/
|
||||
venv/
|
||||
|
||||
# Environment variables and secrets
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# Test and tooling caches
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# IDE and operating system
|
||||
.vscode/
|
||||
.idea/
|
||||
.DS_Store
|
||||
|
||||
# Local audio files
|
||||
audios/
|
||||
*.pcm
|
||||
|
||||
# Temporary files
|
||||
*.log
|
||||
*.tmp
|
||||
1
.python-version
Normal file
1
.python-version
Normal file
@@ -0,0 +1 @@
|
||||
3.12
|
||||
0
Dockerfile
Normal file
0
Dockerfile
Normal file
18
config/development.yaml
Normal file
18
config/development.yaml
Normal file
@@ -0,0 +1,18 @@
|
||||
stt:
|
||||
provider: nvidia
|
||||
options:
|
||||
server: "127.0.0.1:9000"
|
||||
language_code: "pt-BR"
|
||||
automatic_punctuation: true
|
||||
|
||||
tts:
|
||||
provider: voxcpm
|
||||
options:
|
||||
server: "http://127.0.0.1:8000"
|
||||
model: "openbmb/VoxCPM2"
|
||||
voice: "default"
|
||||
|
||||
agent:
|
||||
provider: mock
|
||||
options: {}
|
||||
|
||||
0
config/production.yaml
Normal file
0
config/production.yaml
Normal file
29
config/providers.yaml
Normal file
29
config/providers.yaml
Normal file
@@ -0,0 +1,29 @@
|
||||
providers:
|
||||
|
||||
stt:
|
||||
|
||||
nvidia:
|
||||
name: "NVIDIA ASR"
|
||||
implementation: "voice_assistant.stt.nvidia:NvidiaSTTAdapter"
|
||||
capabilities:
|
||||
realtime: true
|
||||
streaming: true
|
||||
|
||||
tts:
|
||||
|
||||
voxcpm:
|
||||
name: "VoxCPM"
|
||||
implementation: "voice_assistant.tts.voxcpm:VoxCPMTTSAdapter"
|
||||
capabilities:
|
||||
realtime: true
|
||||
streaming: true
|
||||
|
||||
agent:
|
||||
|
||||
mock:
|
||||
name: "Mock Agent"
|
||||
implementation: "voice_assistant.agent.mock:MockAgentAdapter"
|
||||
capabilities:
|
||||
realtime: false
|
||||
streaming: false
|
||||
|
||||
0
config/staging.yaml
Normal file
0
config/staging.yaml
Normal file
29
pyproject.toml
Normal file
29
pyproject.toml
Normal file
@@ -0,0 +1,29 @@
|
||||
[project]
|
||||
name = "voice-assistant"
|
||||
version = "0.1.0"
|
||||
description = "Add your description here"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"fastapi>=0.141.1",
|
||||
"httpx>=0.28,<1",
|
||||
"httpx2>=2.13.1",
|
||||
"ipykernel>=7.3.0",
|
||||
"nvidia-riva-client==2.26.0",
|
||||
"pytest>=9.1.1",
|
||||
"python-dotenv>=1.2.3",
|
||||
"requests>=2.34.2",
|
||||
"uvicorn[standard]>=0.53.0",
|
||||
"websockets>=15.0.1,<16",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
voice-assistant = "voice_assistant:main"
|
||||
|
||||
[build-system]
|
||||
requires = ["uv_build>=0.12.17,<0.13.0"]
|
||||
build-backend = "uv_build"
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
pythonpath = ["src"]
|
||||
2
src/voice_assistant/__init__.py
Normal file
2
src/voice_assistant/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
def main() -> None:
|
||||
print("Hello from voice-assistant!")
|
||||
7
src/voice_assistant/agent/__init__.py
Normal file
7
src/voice_assistant/agent/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
from .adapter import AgentAdapter
|
||||
from .mock import MockAgentAdapter
|
||||
|
||||
__all__ = [
|
||||
"AgentAdapter",
|
||||
"MockAgentAdapter",
|
||||
]
|
||||
14
src/voice_assistant/agent/adapter.py
Normal file
14
src/voice_assistant/agent/adapter.py
Normal file
@@ -0,0 +1,14 @@
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class AgentAdapter(ABC):
|
||||
@abstractmethod
|
||||
async def process(
|
||||
self,
|
||||
session_id: str,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> str:
|
||||
"""Process the final text of a turn and return the response."""
|
||||
raise NotImplementedError
|
||||
|
||||
24
src/voice_assistant/agent/mock.py
Normal file
24
src/voice_assistant/agent/mock.py
Normal file
@@ -0,0 +1,24 @@
|
||||
from .adapter import AgentAdapter
|
||||
|
||||
|
||||
class MockAgentAdapter(AgentAdapter):
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[
|
||||
tuple[str, int, str]
|
||||
] = []
|
||||
|
||||
async def process(
|
||||
self,
|
||||
session_id: str,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> str:
|
||||
self.calls.append(
|
||||
(
|
||||
session_id,
|
||||
turn_id,
|
||||
text,
|
||||
)
|
||||
)
|
||||
|
||||
return f"Resposta simulada para: {text}"
|
||||
0
src/voice_assistant/audio/__init__.py
Normal file
0
src/voice_assistant/audio/__init__.py
Normal file
63
src/voice_assistant/audio/models.py
Normal file
63
src/voice_assistant/audio/models.py
Normal file
@@ -0,0 +1,63 @@
|
||||
"""
|
||||
Canonical audio models for the Voice Assistant.
|
||||
|
||||
Purpose:
|
||||
Define the provider-independent audio format used internally by
|
||||
the Voice Core.
|
||||
|
||||
Responsibilities:
|
||||
- Describe an audio format in a typed structure.
|
||||
- Define the canonical audio format used by the Voice Core.
|
||||
- Provide a shared audio contract for transports and providers.
|
||||
|
||||
Non-responsibilities:
|
||||
- Encoding or decoding audio.
|
||||
- Resampling audio.
|
||||
- Converting provider-specific audio formats.
|
||||
- Managing WebSocket audio frames.
|
||||
- Performing STT or TTS operations.
|
||||
|
||||
Architecture:
|
||||
Client / Transport
|
||||
|
|
||||
v
|
||||
Canonical Audio
|
||||
PCM S16LE / 16 kHz / mono
|
||||
|
|
||||
v
|
||||
Voice Core
|
||||
|
|
||||
+-----+-----+
|
||||
| |
|
||||
v v
|
||||
STT TTS
|
||||
Adapter Adapter
|
||||
|
||||
Provider-specific format conversion, when required, belongs
|
||||
inside the corresponding provider adapter.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class AudioFormat(BaseModel):
|
||||
"""
|
||||
Describe one PCM audio format.
|
||||
|
||||
This model is independent from WebSocket, STT, TTS, or any
|
||||
concrete provider.
|
||||
"""
|
||||
|
||||
encoding: Literal["pcm_s16le"]
|
||||
sample_rate: int
|
||||
channels: int
|
||||
|
||||
|
||||
CANONICAL_AUDIO_FORMAT = AudioFormat(
|
||||
encoding="pcm_s16le",
|
||||
sample_rate=16000,
|
||||
channels=1,
|
||||
)
|
||||
|
||||
0
src/voice_assistant/audio_gateway/__init__.py
Normal file
0
src/voice_assistant/audio_gateway/__init__.py
Normal file
0
src/voice_assistant/audio_gateway/app/__init__.py
Normal file
0
src/voice_assistant/audio_gateway/app/__init__.py
Normal file
914
src/voice_assistant/audio_gateway/app/main.py
Normal file
914
src/voice_assistant/audio_gateway/app/main.py
Normal file
@@ -0,0 +1,914 @@
|
||||
"""
|
||||
Audio Gateway application and WebSocket transport for the Voice Assistant.
|
||||
|
||||
Purpose:
|
||||
Expose the realtime audio protocol and connect client sessions
|
||||
to the VoicePipeline orchestration layer.
|
||||
|
||||
Responsibilities:
|
||||
- Accept and validate WebSocket protocol messages.
|
||||
- Receive canonical PCM audio frames.
|
||||
- Create and configure session-scoped components.
|
||||
- Forward STT events and TTS audio to connected clients.
|
||||
- Delegate semantic turn lifecycle to VoicePipeline.
|
||||
- Coordinate transport cleanup on disconnect.
|
||||
- Report protocol and cleanup errors.
|
||||
|
||||
Non-responsibilities:
|
||||
- Implementing STT, Agent, or TTS provider logic.
|
||||
- Managing semantic turn state directly.
|
||||
- Loading provider implementations dynamically.
|
||||
- Implementing provider-specific cancellation.
|
||||
|
||||
Architecture:
|
||||
WebSocket Client
|
||||
|
|
||||
v
|
||||
Audio Gateway
|
||||
|
|
||||
v
|
||||
VoicePipeline
|
||||
| | |
|
||||
STT Agent TTS
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||
from pydantic import BaseModel, ValidationError
|
||||
import os
|
||||
|
||||
import asyncio
|
||||
|
||||
from voice_assistant.voice_pipeline import VoicePipeline
|
||||
from voice_assistant.stt.adapter import STTAdapter
|
||||
from voice_assistant.tts.adapter import TTSAdapter
|
||||
from voice_assistant.turn_manager import (
|
||||
TurnManager,
|
||||
TurnState,
|
||||
)
|
||||
from voice_assistant.agent.adapter import AgentAdapter
|
||||
|
||||
from voice_assistant.audio.models import (
|
||||
CANONICAL_AUDIO_FORMAT,
|
||||
)
|
||||
|
||||
from voice_assistant.providers import (
|
||||
create_agent_adapter,
|
||||
create_stt_adapter,
|
||||
create_tts_adapter,
|
||||
)
|
||||
|
||||
|
||||
from voice_assistant.config.loader import (
|
||||
load_config,
|
||||
load_provider_catalog,
|
||||
)
|
||||
|
||||
|
||||
from voice_assistant.audio_gateway.app.protocol import (
|
||||
PROTOCOL_VERSION,
|
||||
AUDIO_ENCODING,
|
||||
AUDIO_SAMPLE_RATE,
|
||||
AUDIO_CHANNELS,
|
||||
AudioChunk,
|
||||
AudioStart,
|
||||
AudioStop,
|
||||
Interrupt,
|
||||
SessionEnd,
|
||||
SessionStart,
|
||||
error_event,
|
||||
AgentResponse,
|
||||
Interrupted,
|
||||
SessionEnded,
|
||||
SessionReady,
|
||||
STTFinal,
|
||||
STTPartial,
|
||||
TurnEnded,
|
||||
TurnStarted,
|
||||
)
|
||||
|
||||
from voice_assistant.audio_gateway.app.session import SessionState
|
||||
|
||||
# logger configuration
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI(
|
||||
title="Voice Assistant Audio Gateway",
|
||||
version=PROTOCOL_VERSION,
|
||||
)
|
||||
|
||||
app_config = load_config()
|
||||
provider_catalog = load_provider_catalog()
|
||||
|
||||
MAX_AUDIO_CHUNK_BYTES = 64 * 1024 # 64 KB
|
||||
|
||||
|
||||
|
||||
def create_turn_manager() -> TurnManager:
|
||||
"""Create the turn manager used by the audio gateway."""
|
||||
return TurnManager()
|
||||
|
||||
|
||||
async def send_json(
|
||||
websocket: WebSocket,
|
||||
payload: BaseModel | dict,
|
||||
) -> None:
|
||||
"""
|
||||
Serialize and send one protocol event through the WebSocket.
|
||||
|
||||
Typed Pydantic protocol models are preferred. Dictionary support
|
||||
remains temporarily available for error events until ErrorEvent
|
||||
is migrated to the typed protocol.
|
||||
"""
|
||||
if isinstance(payload, BaseModel):
|
||||
data = payload.model_dump()
|
||||
else:
|
||||
data = payload
|
||||
|
||||
await websocket.send_text(
|
||||
json.dumps(
|
||||
data,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
async def forward_stt_events(
|
||||
websocket: WebSocket,
|
||||
stt: STTAdapter,
|
||||
pipeline: VoicePipeline,
|
||||
session_id: str,
|
||||
turn_id: int,
|
||||
) -> None:
|
||||
async for event in stt.events():
|
||||
if not pipeline.is_active_turn(turn_id):
|
||||
logger.debug(
|
||||
"Discarding stale STT event. session_id=%s turn_id=%s",
|
||||
session_id,
|
||||
turn_id,
|
||||
)
|
||||
break
|
||||
if event.type == "partial":
|
||||
pipeline.on_partial(
|
||||
turn_id,
|
||||
event.text,
|
||||
)
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
STTPartial(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=event.text,
|
||||
),
|
||||
)
|
||||
|
||||
elif event.type == "final":
|
||||
pipeline.on_final(
|
||||
turn_id,
|
||||
event.text,
|
||||
)
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
STTFinal(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=event.text,
|
||||
),
|
||||
)
|
||||
|
||||
result = await pipeline.process_transcript(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=event.text,
|
||||
)
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
AgentResponse(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=result.response_text,
|
||||
),
|
||||
)
|
||||
|
||||
async for audio_chunk in pipeline.synthesize_response(
|
||||
result.response_text,
|
||||
):
|
||||
await websocket.send_bytes(audio_chunk)
|
||||
|
||||
pipeline.on_tts_complete(turn_id)
|
||||
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/ready")
|
||||
async def ready() -> dict[str, str]:
|
||||
return {"status": "ready"}
|
||||
|
||||
|
||||
@app.websocket("/ws/audio")
|
||||
async def audio_socket(websocket: WebSocket) -> None:
|
||||
await websocket.accept()
|
||||
|
||||
session: SessionState | None = None
|
||||
pending_audio_chunk: AudioChunk | None = None
|
||||
|
||||
stt: STTAdapter | None = None
|
||||
stt_events_task: asyncio.Task | None = None
|
||||
|
||||
turn_manager = create_turn_manager()
|
||||
agent: AgentAdapter | None = None
|
||||
|
||||
tts: TTSAdapter | None = None
|
||||
pipeline: VoicePipeline | None = None
|
||||
|
||||
try:
|
||||
while True:
|
||||
message = await websocket.receive()
|
||||
|
||||
if message.get("type") == "websocket.disconnect":
|
||||
break
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Binary WebSocket frame -> PCM
|
||||
# ---------------------------------------------------------
|
||||
if message.get("bytes") is not None:
|
||||
if session is None:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"SESSION_NOT_FOUND",
|
||||
"session.start is required before audio.",
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
if pending_audio_chunk is None:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"INVALID_AUDIO_FRAME",
|
||||
"Binary PCM frame must be preceded by audio.chunk.",
|
||||
session_id=session.session_id,
|
||||
turn_id=session.active_turn_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
audio_payload = message["bytes"]
|
||||
|
||||
if len(audio_payload) > MAX_AUDIO_CHUNK_BYTES:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"MESSAGE_TOO_LARGE",
|
||||
(
|
||||
"Audio chunk exceeds maximum size of "
|
||||
f"{MAX_AUDIO_CHUNK_BYTES} bytes."
|
||||
),
|
||||
session_id=session.session_id,
|
||||
turn_id=pending_audio_chunk.turn_id,
|
||||
details={
|
||||
"max_bytes": MAX_AUDIO_CHUNK_BYTES,
|
||||
"received_bytes": len(audio_payload),
|
||||
},
|
||||
),
|
||||
)
|
||||
pending_audio_chunk = None
|
||||
continue
|
||||
|
||||
try:
|
||||
session.accept_audio_chunk(
|
||||
pending_audio_chunk.turn_id,
|
||||
pending_audio_chunk.sequence_number,
|
||||
len(audio_payload),
|
||||
)
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"AUDIO_SEQUENCE_ERROR",
|
||||
str(exc),
|
||||
session_id=session.session_id,
|
||||
turn_id=pending_audio_chunk.turn_id,
|
||||
details={
|
||||
"expected_sequence_number":
|
||||
session.expected_audio_sequence,
|
||||
},
|
||||
),
|
||||
)
|
||||
pending_audio_chunk = None
|
||||
continue
|
||||
|
||||
# echo do PCM para validar o transporte.
|
||||
if stt is None:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"STT_NOT_STARTED",
|
||||
"audio.start is required before audio.",
|
||||
session_id=session.session_id,
|
||||
turn_id=session.active_turn_id,
|
||||
),
|
||||
)
|
||||
pending_audio_chunk = None
|
||||
continue
|
||||
|
||||
try:
|
||||
await stt.send_audio(audio_payload)
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"STT_AUDIO_ERROR",
|
||||
str(exc),
|
||||
session_id=session.session_id,
|
||||
turn_id=pending_audio_chunk.turn_id,
|
||||
retryable=True,
|
||||
),
|
||||
)
|
||||
pending_audio_chunk = None
|
||||
continue
|
||||
|
||||
pending_audio_chunk = None
|
||||
continue
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Text WebSocket frame -> JSON control event
|
||||
# ---------------------------------------------------------
|
||||
text = message.get("text")
|
||||
|
||||
if text is None:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"INVALID_MESSAGE",
|
||||
"Unsupported WebSocket frame.",
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"INVALID_MESSAGE",
|
||||
"Message must contain valid JSON.",
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
event_type = payload.get("type")
|
||||
|
||||
if not isinstance(event_type, str):
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"INVALID_MESSAGE",
|
||||
"Field 'type' is required.",
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# session.start
|
||||
# ---------------------------------------------------------
|
||||
if event_type == "session.start":
|
||||
if session is not None:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"SESSION_ALREADY_STARTED",
|
||||
"This WebSocket already has an active session.",
|
||||
session_id=session.session_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
try:
|
||||
request = SessionStart.model_validate(payload)
|
||||
except ValidationError as exc:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"INVALID_MESSAGE",
|
||||
"Invalid session.start payload.",
|
||||
details={"errors": exc.errors()},
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
audio = request.audio
|
||||
|
||||
if (
|
||||
audio.encoding != AUDIO_ENCODING
|
||||
or audio.sample_rate != AUDIO_SAMPLE_RATE
|
||||
or audio.channels != AUDIO_CHANNELS
|
||||
):
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"INVALID_AUDIO_FORMAT",
|
||||
(
|
||||
"Only PCM S16LE, mono, 16 kHz "
|
||||
"is supported in the MVP."
|
||||
),
|
||||
details={
|
||||
"expected": {
|
||||
"encoding": AUDIO_ENCODING,
|
||||
"sample_rate": AUDIO_SAMPLE_RATE,
|
||||
"channels": AUDIO_CHANNELS,
|
||||
}
|
||||
},
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
session = SessionState(
|
||||
session_id=request.session_id or SessionState().session_id
|
||||
)
|
||||
|
||||
agent = create_agent_adapter(
|
||||
config=app_config,
|
||||
catalog=provider_catalog,
|
||||
)
|
||||
|
||||
tts = create_tts_adapter(
|
||||
config=app_config,
|
||||
catalog=provider_catalog,
|
||||
)
|
||||
|
||||
pipeline = VoicePipeline(
|
||||
session=session,
|
||||
turn_manager=turn_manager,
|
||||
agent=agent,
|
||||
tts=tts,
|
||||
)
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
SessionReady(
|
||||
protocol_version=PROTOCOL_VERSION,
|
||||
session_id=session.session_id,
|
||||
audio=CANONICAL_AUDIO_FORMAT,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Todos os eventos abaixo exigem sessão ativa
|
||||
# ---------------------------------------------------------
|
||||
if session.closed:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"SESSION_ALREADY_CLOSED",
|
||||
"This session is already closed.",
|
||||
session_id=session.session_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# audio.start
|
||||
# ---------------------------------------------------------
|
||||
if event_type == "audio.start":
|
||||
turn_id: int | None = None
|
||||
|
||||
try:
|
||||
if pipeline is None:
|
||||
raise RuntimeError(
|
||||
"Voice pipeline is not initialized."
|
||||
)
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
turn_id = turn.turn_id
|
||||
|
||||
# Each turn receives a fresh STT provider instance.
|
||||
stt = create_stt_adapter(
|
||||
config=app_config,
|
||||
catalog=provider_catalog,
|
||||
)
|
||||
|
||||
await stt.start_session()
|
||||
|
||||
stt_events_task = asyncio.create_task(
|
||||
forward_stt_events(
|
||||
websocket=websocket,
|
||||
stt=stt,
|
||||
pipeline=pipeline,
|
||||
session_id=session.session_id,
|
||||
turn_id=turn_id,
|
||||
)
|
||||
)
|
||||
pipeline.register_turn_task(stt_events_task)
|
||||
|
||||
|
||||
except (ValidationError, RuntimeError, ValueError) as exc:
|
||||
# Preserve the original startup failure.
|
||||
startup_error = exc
|
||||
cleanup_complete = True
|
||||
|
||||
# Cancel and await any task created during startup.
|
||||
if stt_events_task is not None:
|
||||
task = stt_events_task
|
||||
|
||||
try:
|
||||
if (
|
||||
pipeline is not None
|
||||
and pipeline.turn_task is task
|
||||
):
|
||||
await pipeline.cancel_turn_task()
|
||||
else:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(task),
|
||||
timeout=5.0,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
if not task.cancelled():
|
||||
raise
|
||||
|
||||
except Exception:
|
||||
cleanup_complete = False
|
||||
logger.exception(
|
||||
"Failed to cancel task during "
|
||||
"audio.start rollback."
|
||||
)
|
||||
|
||||
if task.done():
|
||||
stt_events_task = None
|
||||
else:
|
||||
cleanup_complete = False
|
||||
|
||||
# Abort the partially initialized STT provider.
|
||||
if stt is not None:
|
||||
try:
|
||||
await stt.abort_session()
|
||||
except Exception:
|
||||
cleanup_complete = False
|
||||
logger.exception(
|
||||
"Failed to abort STT during "
|
||||
"audio.start rollback."
|
||||
)
|
||||
|
||||
# Only release semantic state after execution
|
||||
# has stopped and provider cleanup succeeded.
|
||||
if cleanup_complete:
|
||||
if turn_id is not None and pipeline is not None:
|
||||
pipeline.interrupt_turn(turn_id)
|
||||
|
||||
stt = None
|
||||
pending_audio_chunk = None
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"STT_START_FAILED",
|
||||
str(startup_error),
|
||||
session_id=session.session_id,
|
||||
turn_id=turn_id,
|
||||
retryable=cleanup_complete,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
# Startup completed successfully.
|
||||
await send_json(
|
||||
websocket,
|
||||
TurnStarted(
|
||||
session_id=session.session_id,
|
||||
turn_id=turn_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# audio.chunk
|
||||
# ---------------------------------------------------------
|
||||
if event_type == "audio.chunk":
|
||||
try:
|
||||
request = AudioChunk.model_validate(payload)
|
||||
|
||||
if session.active_turn_id != request.turn_id:
|
||||
raise ValueError(
|
||||
f"Unexpected turn_id={request.turn_id}; "
|
||||
f"active_turn_id={session.active_turn_id}"
|
||||
)
|
||||
|
||||
if pending_audio_chunk is not None:
|
||||
raise ValueError(
|
||||
"Previous audio chunk is still pending PCM data."
|
||||
)
|
||||
|
||||
pending_audio_chunk = request
|
||||
|
||||
except (ValidationError, ValueError) as exc:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"AUDIO_SEQUENCE_ERROR",
|
||||
str(exc),
|
||||
session_id=session.session_id,
|
||||
turn_id=payload.get("turn_id"),
|
||||
details={
|
||||
"expected_sequence_number":
|
||||
session.expected_audio_sequence,
|
||||
},
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
continue
|
||||
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# audio.stop
|
||||
# ---------------------------------------------------------
|
||||
if event_type == "audio.stop":
|
||||
try:
|
||||
request = AudioStop.model_validate(payload)
|
||||
|
||||
if stt is None:
|
||||
raise RuntimeError(
|
||||
"STT session has not been started."
|
||||
)
|
||||
|
||||
if pending_audio_chunk is not None:
|
||||
raise ValueError(
|
||||
"audio.stop received while an audio chunk is pending."
|
||||
)
|
||||
|
||||
if agent is None:
|
||||
raise RuntimeError(
|
||||
"Agent adapter is not initialized."
|
||||
)
|
||||
|
||||
if tts is None:
|
||||
raise RuntimeError(
|
||||
"TTS adapter is not initialized."
|
||||
)
|
||||
|
||||
if pipeline is None:
|
||||
raise RuntimeError(
|
||||
"Voice pipeline is not initialized."
|
||||
)
|
||||
|
||||
# Finish the STT session normally.
|
||||
await stt.stop_session()
|
||||
|
||||
await pipeline.wait_turn_task()
|
||||
stt_events_task = None
|
||||
|
||||
try:
|
||||
pipeline.complete_turn(request.turn_id)
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
raise RuntimeError(
|
||||
"Turn pipeline did not complete successfully."
|
||||
) from exc
|
||||
|
||||
except (ValidationError, RuntimeError, ValueError) as exc:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"INVALID_TURN",
|
||||
str(exc),
|
||||
session_id=session.session_id,
|
||||
turn_id=payload.get("turn_id"),
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
stt = None
|
||||
pending_audio_chunk = None
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
TurnEnded(
|
||||
session_id=session.session_id,
|
||||
turn_id=request.turn_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# interrupt
|
||||
# ---------------------------------------------------------
|
||||
if event_type == "interrupt":
|
||||
try:
|
||||
request = Interrupt.model_validate(payload)
|
||||
|
||||
except ValidationError as exc:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"INVALID_MESSAGE",
|
||||
"Invalid interrupt payload.",
|
||||
session_id=session.session_id,
|
||||
details={"errors": exc.errors()},
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
if pipeline is None:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"INVALID_TURN",
|
||||
"Voice pipeline is not initialized.",
|
||||
session_id=session.session_id,
|
||||
turn_id=request.turn_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
# Only cancel resources belonging to the requested active turn.
|
||||
if pipeline.is_active_turn(request.turn_id):
|
||||
try:
|
||||
# Cancel and await STT/Agent/TTS execution before
|
||||
# confirming the interruption to the client.
|
||||
await pipeline.cancel_turn_task()
|
||||
|
||||
# Abort transcription without requesting a final result.
|
||||
if stt is not None:
|
||||
await stt.abort_session()
|
||||
|
||||
# Release semantic turn state after execution stops.
|
||||
pipeline.interrupt_if_active(request.turn_id)
|
||||
|
||||
# Clear references to the interrupted execution.
|
||||
stt_events_task = None
|
||||
pending_audio_chunk = None
|
||||
|
||||
except Exception as exc:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"TURN_CANCEL_FAILED",
|
||||
str(exc),
|
||||
session_id=session.session_id,
|
||||
turn_id=request.turn_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
else:
|
||||
# Preserve the existing tolerant behavior for unknown turns.
|
||||
pipeline.interrupt_if_active(request.turn_id)
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
Interrupted(
|
||||
session_id=session.session_id,
|
||||
turn_id=request.turn_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# session.end
|
||||
# ---------------------------------------------------------
|
||||
if event_type == "session.end":
|
||||
try:
|
||||
SessionEnd.model_validate(payload)
|
||||
|
||||
if pipeline is None:
|
||||
raise RuntimeError(
|
||||
"Voice pipeline is not initialized."
|
||||
)
|
||||
|
||||
# Cancel and await any running STT/Agent/TTS task.
|
||||
await pipeline.cancel_turn_task()
|
||||
|
||||
# Abort STT without waiting for a final transcript.
|
||||
if stt is not None:
|
||||
await stt.abort_session()
|
||||
|
||||
# Close semantic turn and session state.
|
||||
pipeline.close_session()
|
||||
|
||||
# Release Gateway-owned references.
|
||||
stt_events_task = None
|
||||
stt = None
|
||||
pending_audio_chunk = None
|
||||
|
||||
except (ValidationError, RuntimeError, ValueError) as exc:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"SESSION_END_FAILED",
|
||||
str(exc),
|
||||
session_id=session.session_id,
|
||||
turn_id=session.active_turn_id,
|
||||
retryable=True,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
SessionEnded(
|
||||
session_id=session.session_id,
|
||||
),
|
||||
)
|
||||
break
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Evento desconhecido
|
||||
# ---------------------------------------------------------
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"UNSUPPORTED_EVENT",
|
||||
f"Unsupported event type: {event_type!r}.",
|
||||
session_id=session.session_id,
|
||||
turn_id=session.active_turn_id,
|
||||
),
|
||||
)
|
||||
|
||||
except WebSocketDisconnect:
|
||||
# The finally block owns connection cleanup.
|
||||
pass
|
||||
|
||||
finally:
|
||||
# Cancel and await the task owned by VoicePipeline.
|
||||
if pipeline is not None:
|
||||
try:
|
||||
await pipeline.cancel_turn_task()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to cancel voice turn task. session_id=%s",
|
||||
session.session_id if session else None,
|
||||
)
|
||||
|
||||
# A task may have been created before registration failed.
|
||||
# Handle it independently if the pipeline does not own it.
|
||||
if stt_events_task is not None:
|
||||
task = stt_events_task
|
||||
|
||||
if (
|
||||
pipeline is None
|
||||
or pipeline.turn_task is not task
|
||||
):
|
||||
try:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(task),
|
||||
timeout=5.0,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
if not task.cancelled():
|
||||
raise
|
||||
|
||||
except TimeoutError:
|
||||
logger.error(
|
||||
"Unregistered STT task cleanup timed out. "
|
||||
"session_id=%s",
|
||||
session.session_id if session else None,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to clean up unregistered STT task. "
|
||||
"session_id=%s",
|
||||
session.session_id if session else None,
|
||||
)
|
||||
|
||||
if task.done():
|
||||
stt_events_task = None
|
||||
# Abort STT without waiting for a final transcript.
|
||||
if stt is not None:
|
||||
try:
|
||||
await stt.abort_session()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to abort STT session. session_id=%s",
|
||||
session.session_id if session else None,
|
||||
)
|
||||
|
||||
# Release session and semantic turn state.
|
||||
try:
|
||||
if pipeline is not None:
|
||||
pipeline.close_session()
|
||||
elif session is not None:
|
||||
session.close()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to close voice session. session_id=%s",
|
||||
session.session_id if session else None,
|
||||
)
|
||||
|
||||
163
src/voice_assistant/audio_gateway/app/protocol.py
Normal file
163
src/voice_assistant/audio_gateway/app/protocol.py
Normal file
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from voice_assistant.audio.models import (
|
||||
AudioFormat,
|
||||
CANONICAL_AUDIO_FORMAT,
|
||||
)
|
||||
|
||||
PROTOCOL_VERSION = "1.0"
|
||||
AUDIO_ENCODING = "pcm_s16le"
|
||||
AUDIO_SAMPLE_RATE = 16000
|
||||
AUDIO_CHANNELS = 1
|
||||
|
||||
|
||||
class SessionStart(BaseModel):
|
||||
"""Request that initializes a new Voice Assistant session."""
|
||||
|
||||
type: Literal["session.start"] = "session.start"
|
||||
session_id: str | None = None
|
||||
|
||||
audio: AudioFormat = Field(
|
||||
default_factory=lambda: CANONICAL_AUDIO_FORMAT.model_copy()
|
||||
)
|
||||
|
||||
|
||||
class ErrorEvent(BaseModel):
|
||||
"""Error response sent by the Audio Gateway."""
|
||||
|
||||
type: Literal["error"] = "error"
|
||||
session_id: str | None = None
|
||||
turn_id: int | None = None
|
||||
code: str
|
||||
message: str
|
||||
retryable: bool = False
|
||||
details: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
)
|
||||
|
||||
|
||||
class AudioStart(BaseModel):
|
||||
"""Request that starts a new audio turn."""
|
||||
|
||||
type: Literal["audio.start"] = "audio.start"
|
||||
|
||||
|
||||
class AudioChunk(BaseModel):
|
||||
"""Metadata describing the next binary audio chunk."""
|
||||
|
||||
type: Literal["audio.chunk"] = "audio.chunk"
|
||||
turn_id: int
|
||||
sequence_number: int
|
||||
|
||||
|
||||
class AudioStop(BaseModel):
|
||||
"""Request that finishes audio input for the active turn."""
|
||||
|
||||
type: Literal["audio.stop"] = "audio.stop"
|
||||
turn_id: int
|
||||
|
||||
|
||||
class Interrupt(BaseModel):
|
||||
"""Request that interrupts the active turn."""
|
||||
|
||||
type: Literal["interrupt"] = "interrupt"
|
||||
turn_id: int
|
||||
|
||||
|
||||
class SessionEnd(BaseModel):
|
||||
"""Request that closes the active session."""
|
||||
|
||||
type: Literal["session.end"] = "session.end"
|
||||
|
||||
|
||||
class SessionReady(BaseModel):
|
||||
"""Response confirming that a session is ready."""
|
||||
|
||||
type: Literal["session.ready"] = "session.ready"
|
||||
protocol_version: str
|
||||
session_id: str
|
||||
audio: AudioFormat
|
||||
|
||||
|
||||
class TurnStarted(BaseModel):
|
||||
"""Response confirming that a new turn has started."""
|
||||
|
||||
type: Literal["turn.started"] = "turn.started"
|
||||
session_id: str
|
||||
turn_id: int
|
||||
|
||||
|
||||
class STTPartial(BaseModel):
|
||||
"""Partial transcription produced during a turn."""
|
||||
|
||||
type: Literal["stt.partial"] = "stt.partial"
|
||||
session_id: str
|
||||
turn_id: int
|
||||
text: str
|
||||
|
||||
|
||||
class STTFinal(BaseModel):
|
||||
"""Final transcription produced for a turn."""
|
||||
|
||||
type: Literal["stt.final"] = "stt.final"
|
||||
session_id: str
|
||||
turn_id: int
|
||||
text: str
|
||||
|
||||
|
||||
class AgentResponse(BaseModel):
|
||||
"""Text response produced by the configured Agent."""
|
||||
|
||||
type: Literal["agent.response"] = "agent.response"
|
||||
session_id: str
|
||||
turn_id: int
|
||||
text: str
|
||||
|
||||
|
||||
class TurnEnded(BaseModel):
|
||||
"""Response confirming successful completion of a turn."""
|
||||
|
||||
type: Literal["turn.ended"] = "turn.ended"
|
||||
session_id: str
|
||||
turn_id: int
|
||||
|
||||
|
||||
class Interrupted(BaseModel):
|
||||
"""Response confirming interruption of a turn."""
|
||||
|
||||
type: Literal["interrupted"] = "interrupted"
|
||||
session_id: str
|
||||
turn_id: int
|
||||
|
||||
|
||||
class SessionEnded(BaseModel):
|
||||
"""Response confirming that a session has ended."""
|
||||
|
||||
type: Literal["session.ended"] = "session.ended"
|
||||
session_id: str
|
||||
|
||||
|
||||
def error_event(
|
||||
code: str,
|
||||
message: str,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
turn_id: int | None = None,
|
||||
retryable: bool = False,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> ErrorEvent:
|
||||
"""Build a typed error protocol event."""
|
||||
|
||||
return ErrorEvent(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
code=code,
|
||||
message=message,
|
||||
retryable=retryable,
|
||||
details=details or {},
|
||||
)
|
||||
|
||||
70
src/voice_assistant/audio_gateway/app/session.py
Normal file
70
src/voice_assistant/audio_gateway/app/session.py
Normal file
@@ -0,0 +1,70 @@
|
||||
from dataclasses import dataclass, field
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
@dataclass
|
||||
class SessionState:
|
||||
session_id: str = field(default_factory=lambda: str(uuid4()))
|
||||
active_turn_id: int | None = None
|
||||
next_turn_id: int = 1
|
||||
expected_audio_sequence: int = 0
|
||||
closed: bool = False
|
||||
|
||||
def start_turn(self) -> int:
|
||||
if self.closed:
|
||||
raise RuntimeError("Session is closed.")
|
||||
|
||||
if self.active_turn_id is not None:
|
||||
raise RuntimeError("A turn is already active.")
|
||||
|
||||
self.active_turn_id = self.next_turn_id
|
||||
self.next_turn_id += 1
|
||||
self.expected_audio_sequence = 0
|
||||
|
||||
return self.active_turn_id
|
||||
|
||||
def accept_audio_chunk(
|
||||
self,
|
||||
turn_id: int,
|
||||
sequence_number: int,
|
||||
payload_size: int,
|
||||
) -> None:
|
||||
if self.closed:
|
||||
raise RuntimeError("Session is closed.")
|
||||
|
||||
if self.active_turn_id != turn_id:
|
||||
raise ValueError(
|
||||
f"Unexpected turn_id={turn_id}; "
|
||||
f"active_turn_id={self.active_turn_id}."
|
||||
)
|
||||
|
||||
if sequence_number != self.expected_audio_sequence:
|
||||
raise ValueError(
|
||||
f"Expected sequence_number="
|
||||
f"{self.expected_audio_sequence}, "
|
||||
f"received={sequence_number}."
|
||||
)
|
||||
|
||||
if payload_size == 0:
|
||||
raise ValueError("Audio chunk cannot be empty.")
|
||||
|
||||
self.expected_audio_sequence += 1
|
||||
|
||||
def end_turn(self, turn_id: int) -> None:
|
||||
if self.active_turn_id != turn_id:
|
||||
raise ValueError(
|
||||
f"Cannot end turn {turn_id}; "
|
||||
f"active_turn_id={self.active_turn_id}."
|
||||
)
|
||||
|
||||
self.active_turn_id = None
|
||||
self.expected_audio_sequence = 0
|
||||
|
||||
def interrupt(self, turn_id: int) -> None:
|
||||
if self.active_turn_id == turn_id:
|
||||
self.active_turn_id = None
|
||||
self.expected_audio_sequence = 0
|
||||
|
||||
def close(self) -> None:
|
||||
self.active_turn_id = None
|
||||
self.closed = True
|
||||
0
src/voice_assistant/config/__init__.py
Normal file
0
src/voice_assistant/config/__init__.py
Normal file
109
src/voice_assistant/config/loader.py
Normal file
109
src/voice_assistant/config/loader.py
Normal file
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
Configuration loading for the Voice Assistant.
|
||||
|
||||
Purpose:
|
||||
Load environment-specific YAML configuration and the provider
|
||||
catalog, then validate both using the typed configuration models.
|
||||
|
||||
Responsibilities:
|
||||
- Select the active environment.
|
||||
- Load environment configuration.
|
||||
- Load provider catalog metadata.
|
||||
- Validate configuration through Pydantic models.
|
||||
|
||||
Non-responsibilities:
|
||||
- Instantiating providers.
|
||||
- Executing STT, Agent, or TTS functionality.
|
||||
- Managing application lifecycle.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from .models import AppConfig, ProviderCatalog
|
||||
|
||||
|
||||
DEFAULT_CONFIG_DIR = Path("config")
|
||||
|
||||
|
||||
def get_environment() -> str:
|
||||
"""
|
||||
Return the configured application environment.
|
||||
|
||||
Defaults to development for local execution.
|
||||
"""
|
||||
return os.getenv(
|
||||
"APP_ENV",
|
||||
"development",
|
||||
)
|
||||
|
||||
|
||||
def load_config(
|
||||
environment: str | None = None,
|
||||
config_dir: Path = DEFAULT_CONFIG_DIR,
|
||||
) -> AppConfig:
|
||||
"""Load and validate the selected environment configuration."""
|
||||
|
||||
environment = environment or get_environment()
|
||||
|
||||
config_path = config_dir / f"{environment}.yaml"
|
||||
|
||||
if not config_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Configuration file not found: {config_path}"
|
||||
)
|
||||
|
||||
with config_path.open(
|
||||
"r",
|
||||
encoding="utf-8",
|
||||
) as file:
|
||||
data: Any = yaml.safe_load(file)
|
||||
|
||||
if data is None:
|
||||
raise ValueError(
|
||||
f"Configuration file is empty: {config_path}"
|
||||
)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(
|
||||
f"Configuration root must be a mapping: "
|
||||
f"{config_path}"
|
||||
)
|
||||
|
||||
return AppConfig.model_validate(data)
|
||||
|
||||
|
||||
def load_provider_catalog(
|
||||
config_dir: Path = DEFAULT_CONFIG_DIR,
|
||||
) -> ProviderCatalog:
|
||||
"""Load and validate the provider catalog."""
|
||||
|
||||
config_path = config_dir / "providers.yaml"
|
||||
|
||||
if not config_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Provider catalog not found: {config_path}"
|
||||
)
|
||||
|
||||
with config_path.open(
|
||||
"r",
|
||||
encoding="utf-8",
|
||||
) as file:
|
||||
data: Any = yaml.safe_load(file)
|
||||
|
||||
if data is None:
|
||||
raise ValueError(
|
||||
f"Provider catalog is empty: {config_path}"
|
||||
)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(
|
||||
f"Provider catalog root must be a mapping: "
|
||||
f"{config_path}"
|
||||
)
|
||||
|
||||
return ProviderCatalog.model_validate(data)
|
||||
|
||||
73
src/voice_assistant/config/models.py
Normal file
73
src/voice_assistant/config/models.py
Normal file
@@ -0,0 +1,73 @@
|
||||
"""
|
||||
Typed configuration models for the Voice Assistant.
|
||||
|
||||
Purpose:
|
||||
Define and validate the configuration structures used by the
|
||||
application and by the dynamic provider loading system.
|
||||
|
||||
Responsibilities:
|
||||
- Define the application configuration for STT, Agent, and TTS.
|
||||
- Define the catalog structure for available providers.
|
||||
- Represent provider-specific options without coupling the core
|
||||
configuration model to a specific vendor or implementation.
|
||||
- Describe provider capabilities for discovery and future tooling.
|
||||
|
||||
Non-responsibilities:
|
||||
- Loading YAML files.
|
||||
- Reading environment variables or secrets.
|
||||
- Selecting or instantiating provider implementations.
|
||||
- Executing STT, Agent, or TTS functionality.
|
||||
|
||||
Architecture:
|
||||
YAML
|
||||
|
|
||||
v
|
||||
Config Loader
|
||||
|
|
||||
v
|
||||
AppConfig / ProviderCatalog
|
||||
|
|
||||
v
|
||||
Provider Loader
|
||||
|
|
||||
v
|
||||
STTAdapter / AgentAdapter / TTSAdapter
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ProviderDefinition(BaseModel):
|
||||
"""Metadata describing one available provider."""
|
||||
|
||||
name: str
|
||||
implementation: str
|
||||
capabilities: dict[str, bool] = Field(
|
||||
default_factory=dict,
|
||||
)
|
||||
|
||||
|
||||
class ProviderCatalog(BaseModel):
|
||||
"""Available providers grouped by component type."""
|
||||
|
||||
providers: dict[
|
||||
str,
|
||||
dict[str, ProviderDefinition],
|
||||
] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class ComponentConfig(BaseModel):
|
||||
"""Selected provider and its provider-specific runtime options."""
|
||||
|
||||
provider: str
|
||||
options: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class AppConfig(BaseModel):
|
||||
"""Provider selections for one application environment."""
|
||||
|
||||
stt: ComponentConfig
|
||||
tts: ComponentConfig
|
||||
agent: ComponentConfig
|
||||
187
src/voice_assistant/providers.py
Normal file
187
src/voice_assistant/providers.py
Normal file
@@ -0,0 +1,187 @@
|
||||
"""
|
||||
Dynamic provider loading and composition for the Voice Assistant.
|
||||
|
||||
Purpose:
|
||||
Resolve provider names from the application configuration and
|
||||
instantiate implementations that satisfy the corresponding adapter
|
||||
contract.
|
||||
|
||||
Responsibilities:
|
||||
- Resolve provider implementations from ProviderCatalog metadata.
|
||||
- Import provider classes dynamically.
|
||||
- Validate that the loaded implementation matches the expected adapter.
|
||||
- Instantiate STT, Agent, and TTS providers using provider-specific options.
|
||||
|
||||
Non-responsibilities:
|
||||
- Loading YAML files.
|
||||
- Reading secrets directly.
|
||||
- Managing WebSocket connections.
|
||||
- Managing turn state.
|
||||
- Executing provider functionality.
|
||||
|
||||
Architecture:
|
||||
AppConfig + ProviderCatalog
|
||||
|
|
||||
v
|
||||
Provider Loader
|
||||
|
|
||||
v
|
||||
STTAdapter / AgentAdapter / TTSAdapter
|
||||
|
|
||||
v
|
||||
concrete provider
|
||||
"""
|
||||
|
||||
from importlib import import_module
|
||||
from typing import Any
|
||||
|
||||
from voice_assistant.agent.adapter import AgentAdapter
|
||||
from voice_assistant.config.models import (
|
||||
AppConfig,
|
||||
ComponentConfig,
|
||||
ProviderCatalog,
|
||||
)
|
||||
from voice_assistant.stt.adapter import STTAdapter
|
||||
from voice_assistant.tts.adapter import TTSAdapter
|
||||
|
||||
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
|
||||
|
||||
PROVIDER_CONTRACTS: dict[str, type] = {
|
||||
"stt": STTAdapter,
|
||||
"agent": AgentAdapter,
|
||||
"tts": TTSAdapter,
|
||||
}
|
||||
|
||||
|
||||
def load_provider_class(
|
||||
catalog: ProviderCatalog,
|
||||
component: str,
|
||||
provider: str,
|
||||
) -> type:
|
||||
"""
|
||||
Load and validate a provider class from the provider catalog.
|
||||
|
||||
The catalog stores the import path, while this function ensures that
|
||||
the resolved class implements the adapter contract associated with the
|
||||
requested component.
|
||||
"""
|
||||
contract = PROVIDER_CONTRACTS.get(component)
|
||||
|
||||
if contract is None:
|
||||
raise ValueError(
|
||||
f"Unsupported provider component: {component!r}."
|
||||
)
|
||||
|
||||
component_providers = catalog.providers.get(component)
|
||||
|
||||
if component_providers is None:
|
||||
raise ValueError(
|
||||
f"No providers are registered for component {component!r}."
|
||||
)
|
||||
|
||||
definition = component_providers.get(provider)
|
||||
|
||||
if definition is None:
|
||||
raise ValueError(
|
||||
f"Provider {provider!r} is not registered for "
|
||||
f"component {component!r}."
|
||||
)
|
||||
|
||||
module_name, class_name = definition.implementation.split(
|
||||
":",
|
||||
maxsplit=1,
|
||||
)
|
||||
|
||||
try:
|
||||
module = import_module(module_name)
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
f"Could not import provider module "
|
||||
f"{module_name!r} for {component!r}/{provider!r}."
|
||||
) from exc
|
||||
|
||||
try:
|
||||
provider_class = getattr(module, class_name)
|
||||
except AttributeError as exc:
|
||||
raise RuntimeError(
|
||||
f"Provider class {class_name!r} was not found in "
|
||||
f"module {module_name!r}."
|
||||
) from exc
|
||||
|
||||
if not isinstance(provider_class, type):
|
||||
raise TypeError(
|
||||
f"Provider {component!r}/{provider!r} does not resolve "
|
||||
f"to a class."
|
||||
)
|
||||
|
||||
if not issubclass(provider_class, contract):
|
||||
raise TypeError(
|
||||
f"Provider {component!r}/{provider!r} must implement "
|
||||
f"{contract.__name__}."
|
||||
)
|
||||
|
||||
return provider_class
|
||||
|
||||
|
||||
def _create_provider(
|
||||
catalog: ProviderCatalog,
|
||||
component: str,
|
||||
config: ComponentConfig,
|
||||
) -> Any:
|
||||
"""
|
||||
Instantiate the provider selected by a component configuration.
|
||||
|
||||
Provider-specific constructor arguments are kept inside `options`
|
||||
so the common configuration model does not need to know the API of
|
||||
every vendor.
|
||||
"""
|
||||
provider_class = load_provider_class(
|
||||
catalog=catalog,
|
||||
component=component,
|
||||
provider=config.provider,
|
||||
)
|
||||
|
||||
try:
|
||||
return provider_class(**config.options)
|
||||
except TypeError as exc:
|
||||
raise TypeError(
|
||||
f"Invalid options for provider "
|
||||
f"{component!r}/{config.provider!r}: {exc}"
|
||||
) from exc
|
||||
|
||||
|
||||
def create_stt_adapter(
|
||||
config: AppConfig,
|
||||
catalog: ProviderCatalog,
|
||||
) -> STTAdapter:
|
||||
"""Create the configured STT adapter."""
|
||||
return _create_provider(
|
||||
catalog=catalog,
|
||||
component="stt",
|
||||
config=config.stt,
|
||||
)
|
||||
|
||||
|
||||
def create_agent_adapter(
|
||||
config: AppConfig,
|
||||
catalog: ProviderCatalog,
|
||||
) -> AgentAdapter:
|
||||
"""Create the configured Agent adapter."""
|
||||
return _create_provider(
|
||||
catalog=catalog,
|
||||
component="agent",
|
||||
config=config.agent,
|
||||
)
|
||||
|
||||
|
||||
def create_tts_adapter(
|
||||
config: AppConfig,
|
||||
catalog: ProviderCatalog,
|
||||
) -> TTSAdapter:
|
||||
"""Create the configured TTS adapter."""
|
||||
return _create_provider(
|
||||
catalog=catalog,
|
||||
component="tts",
|
||||
config=config.tts,
|
||||
)
|
||||
|
||||
0
src/voice_assistant/stt/__init__.py
Normal file
0
src/voice_assistant/stt/__init__.py
Normal file
85
src/voice_assistant/stt/adapter.py
Normal file
85
src/voice_assistant/stt/adapter.py
Normal file
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
Provider-independent Speech-to-Text adapter contract.
|
||||
|
||||
Purpose:
|
||||
Define the interface that every STT provider must implement
|
||||
to participate in the Voice Assistant pipeline.
|
||||
|
||||
Responsibilities:
|
||||
- Start transcription sessions.
|
||||
- Receive canonical PCM audio chunks.
|
||||
- Finish transcription sessions normally.
|
||||
- Abort transcription sessions during interruption or cleanup.
|
||||
- Expose provider-independent transcript events.
|
||||
|
||||
Non-responsibilities:
|
||||
- Managing Audio Gateway WebSocket connections.
|
||||
- Managing SessionState or TurnManager.
|
||||
- Executing Agent or TTS operations.
|
||||
- Implementing provider-specific communication.
|
||||
|
||||
Architecture:
|
||||
Audio Gateway / VoicePipeline
|
||||
|
|
||||
v
|
||||
STTAdapter
|
||||
|
|
||||
+-------+-------+
|
||||
| |
|
||||
v v
|
||||
NvidiaSTTAdapter MockSTTAdapter
|
||||
|
||||
Concrete providers implement this contract without exposing
|
||||
their internal protocols to the Voice Core.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from .models import TranscriptEvent
|
||||
|
||||
|
||||
class STTAdapter(ABC):
|
||||
"""Abstract contract for Speech-to-Text providers."""
|
||||
|
||||
@abstractmethod
|
||||
async def start_session(self) -> None:
|
||||
"""Initialize a transcription session."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def send_audio(self, audio: bytes) -> None:
|
||||
"""Send one canonical PCM audio chunk for transcription."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def stop_session(self) -> None:
|
||||
"""
|
||||
Finish transcription normally.
|
||||
|
||||
The provider may wait for its final transcription result
|
||||
before releasing the session.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def abort_session(self) -> None:
|
||||
"""
|
||||
Abort transcription without waiting for a final result.
|
||||
|
||||
Implementations must:
|
||||
- Stop accepting new audio.
|
||||
- Release active provider resources.
|
||||
- Unblock consumers waiting for transcript events.
|
||||
- Avoid waiting for final transcription.
|
||||
- Support repeated calls safely.
|
||||
|
||||
The provider must not emit new transcription results after
|
||||
the abort operation has completed.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def events(self) -> AsyncIterator[TranscriptEvent]:
|
||||
"""Expose the asynchronous stream of transcript events."""
|
||||
raise NotImplementedError
|
||||
161
src/voice_assistant/stt/mock.py
Normal file
161
src/voice_assistant/stt/mock.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""
|
||||
In-memory mock STT provider for the Voice Assistant.
|
||||
|
||||
Purpose:
|
||||
Simulate a realtime STT provider without external services.
|
||||
|
||||
Responsibilities:
|
||||
- Simulate transcription session lifecycle.
|
||||
- Validate incoming audio chunks.
|
||||
- Expose partial and final transcript events.
|
||||
- Support normal stop and immediate abort.
|
||||
- Provide deterministic behavior for automated tests.
|
||||
|
||||
Non-responsibilities:
|
||||
- Performing real speech recognition.
|
||||
- Managing network connections.
|
||||
- Managing voice turns or Gateway sessions.
|
||||
- Executing Agent or TTS operations.
|
||||
|
||||
Architecture:
|
||||
Tests / VoicePipeline
|
||||
|
|
||||
v
|
||||
STTAdapter
|
||||
|
|
||||
v
|
||||
MockSTTAdapter
|
||||
|
|
||||
v
|
||||
Async event queue
|
||||
"""
|
||||
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from .adapter import STTAdapter
|
||||
from .models import TranscriptEvent
|
||||
|
||||
|
||||
class MockSTTAdapter(STTAdapter):
|
||||
def __init__(self) -> None:
|
||||
self._events: asyncio.Queue[
|
||||
TranscriptEvent | None
|
||||
] = asyncio.Queue()
|
||||
|
||||
self._started = False
|
||||
|
||||
async def start_session(self) -> None:
|
||||
self._started = True
|
||||
self._events = asyncio.Queue()
|
||||
|
||||
async def send_audio(self, audio: bytes) -> None:
|
||||
if not self._started:
|
||||
raise RuntimeError(
|
||||
"STT session has not been started."
|
||||
)
|
||||
|
||||
if not audio:
|
||||
raise ValueError(
|
||||
"Audio chunk cannot be empty."
|
||||
)
|
||||
|
||||
async def stop_session(self) -> None:
|
||||
if not self._started:
|
||||
return
|
||||
|
||||
self._started = False
|
||||
|
||||
await self._events.put(None)
|
||||
|
||||
|
||||
async def abort_session(self) -> None:
|
||||
"""
|
||||
Abort transcription and discard pending transcript events.
|
||||
|
||||
Unlike stop_session(), abort_session() must not deliver
|
||||
queued partial or final transcription results.
|
||||
"""
|
||||
if not self._started:
|
||||
return
|
||||
|
||||
# Reject further audio and transcript events.
|
||||
self._started = False
|
||||
|
||||
# Discard queued transcript events from the aborted session.
|
||||
while True:
|
||||
try:
|
||||
self._events.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
|
||||
# Unblock consumers waiting for the next event.
|
||||
await self._events.put(None)
|
||||
|
||||
|
||||
def events(self) -> AsyncIterator[TranscriptEvent]:
|
||||
return self._event_stream()
|
||||
|
||||
async def _event_stream(
|
||||
self,
|
||||
) -> AsyncIterator[TranscriptEvent]:
|
||||
while True:
|
||||
event = await self._events.get()
|
||||
|
||||
if event is None:
|
||||
break
|
||||
|
||||
yield event
|
||||
|
||||
async def emit_partial(self, text: str) -> None:
|
||||
if not self._started:
|
||||
raise RuntimeError(
|
||||
"STT session has not been started."
|
||||
)
|
||||
|
||||
await self._events.put(
|
||||
TranscriptEvent(
|
||||
type="partial",
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
|
||||
async def emit_final(self, text: str) -> None:
|
||||
if not self._started:
|
||||
raise RuntimeError(
|
||||
"STT session has not been started."
|
||||
)
|
||||
|
||||
await self._events.put(
|
||||
TranscriptEvent(
|
||||
type="final",
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
|
||||
def emit_partial_now(self, text: str) -> None:
|
||||
if not self._started:
|
||||
raise RuntimeError(
|
||||
"STT session has not been started."
|
||||
)
|
||||
|
||||
self._events.put_nowait(
|
||||
TranscriptEvent(
|
||||
type="partial",
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
|
||||
def emit_final_now(self, text: str) -> None:
|
||||
if not self._started:
|
||||
raise RuntimeError(
|
||||
"STT session has not been started."
|
||||
)
|
||||
|
||||
self._events.put_nowait(
|
||||
TranscriptEvent(
|
||||
type="final",
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
11
src/voice_assistant/stt/models.py
Normal file
11
src/voice_assistant/stt/models.py
Normal file
@@ -0,0 +1,11 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
|
||||
TranscriptType = Literal["partial", "final"]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class TranscriptEvent:
|
||||
type: TranscriptType
|
||||
text: str
|
||||
430
src/voice_assistant/stt/nvidia.py
Normal file
430
src/voice_assistant/stt/nvidia.py
Normal file
@@ -0,0 +1,430 @@
|
||||
"""
|
||||
NVIDIA realtime STT provider for the Voice Assistant.
|
||||
|
||||
Purpose:
|
||||
Implement the generic STTAdapter contract using NVIDIA Speech NIM.
|
||||
|
||||
Responsibilities:
|
||||
- Create and manage NVIDIA realtime transcription sessions.
|
||||
- Send canonical PCM audio to the NVIDIA ASR service.
|
||||
- Receive provider-specific transcription events.
|
||||
- Translate NVIDIA events into provider-independent TranscriptEvent objects.
|
||||
- Manage provider connection lifecycle and cleanup.
|
||||
|
||||
Non-responsibilities:
|
||||
- Managing WebSocket client connections to the Audio Gateway.
|
||||
- Managing Voice Assistant sessions or turns.
|
||||
- Executing Agent or TTS functionality.
|
||||
- Defining the canonical Voice Core audio format.
|
||||
|
||||
Architecture:
|
||||
Voice Core
|
||||
|
|
||||
| canonical PCM audio
|
||||
v
|
||||
STTAdapter
|
||||
|
|
||||
v
|
||||
NvidiaSTTAdapter
|
||||
|
|
||||
v
|
||||
NVIDIA Speech NIM
|
||||
|
||||
NVIDIA-specific HTTP, WebSocket, session, and event details remain
|
||||
isolated inside this adapter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import websockets
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
|
||||
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
|
||||
|
||||
from .adapter import STTAdapter
|
||||
from .models import TranscriptEvent
|
||||
|
||||
|
||||
class NvidiaSTTAdapter(STTAdapter):
|
||||
"""Realtime STT adapter for NVIDIA Speech NIM."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server: str = "127.0.0.1:9000",
|
||||
language_code: str = "pt-BR",
|
||||
automatic_punctuation: bool = True,
|
||||
) -> None:
|
||||
self.server = server
|
||||
self.language_code = language_code
|
||||
self.automatic_punctuation = automatic_punctuation
|
||||
|
||||
self._http: httpx.AsyncClient | None = None
|
||||
self._websocket: ClientConnection | None = None
|
||||
self._receiver_task: asyncio.Task[None] | None = None
|
||||
|
||||
self._events: asyncio.Queue[TranscriptEvent | None] = asyncio.Queue()
|
||||
|
||||
self._started = False
|
||||
self._stopping = False
|
||||
self._aborted = False
|
||||
self._abort_lock = asyncio.Lock()
|
||||
|
||||
|
||||
async def start_session(self) -> None:
|
||||
if self._started:
|
||||
raise RuntimeError("STT session is already started.")
|
||||
|
||||
self._http = httpx.AsyncClient(timeout=10.0)
|
||||
|
||||
try:
|
||||
session = await self._create_session()
|
||||
self._websocket = await self._connect()
|
||||
|
||||
await self._initialize_session(session)
|
||||
|
||||
self._started = True
|
||||
self._stopping = False
|
||||
self._aborted = False
|
||||
|
||||
self._receiver_task = asyncio.create_task(
|
||||
self._receive_events()
|
||||
)
|
||||
|
||||
except Exception:
|
||||
await self._cleanup()
|
||||
raise
|
||||
|
||||
async def send_audio(self, audio: bytes) -> None:
|
||||
if not self._started or self._websocket is None:
|
||||
raise RuntimeError("STT session has not been started.")
|
||||
|
||||
if not audio:
|
||||
raise ValueError("Audio chunk cannot be empty.")
|
||||
|
||||
await self._send_json(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(audio).decode("utf-8"),
|
||||
}
|
||||
)
|
||||
|
||||
await self._send_json(
|
||||
{
|
||||
"type": "input_audio_buffer.commit",
|
||||
}
|
||||
)
|
||||
|
||||
async def stop_session(self) -> None:
|
||||
if not self._started or self._websocket is None:
|
||||
return
|
||||
|
||||
self._stopping = True
|
||||
|
||||
try:
|
||||
await self._send_json(
|
||||
{
|
||||
"type": "input_audio_buffer.done",
|
||||
}
|
||||
)
|
||||
|
||||
if self._receiver_task is not None:
|
||||
await self._receiver_task
|
||||
|
||||
finally:
|
||||
await self._cleanup()
|
||||
|
||||
|
||||
async def abort_session(self) -> None:
|
||||
"""
|
||||
Abort the NVIDIA transcription session safely.
|
||||
|
||||
Purpose:
|
||||
Stop transcription immediately without waiting for a final
|
||||
transcript, including when multiple abort requests overlap.
|
||||
|
||||
Responsibilities:
|
||||
- Serialize concurrent abort requests.
|
||||
- Stop accepting new audio.
|
||||
- Cancel and await the receiver task.
|
||||
- Release WebSocket and HTTP resources.
|
||||
- Discard pending transcription events.
|
||||
- Publish one end-of-stream marker.
|
||||
|
||||
Non-responsibilities:
|
||||
- Managing Gateway WebSocket connections.
|
||||
- Updating VoicePipeline turn state.
|
||||
- Cancelling Agent or TTS execution.
|
||||
"""
|
||||
async with self._abort_lock:
|
||||
# A completed abort must not run cleanup again.
|
||||
if self._aborted:
|
||||
return
|
||||
|
||||
# Mark the session as aborted before releasing resources.
|
||||
self._aborted = True
|
||||
self._stopping = True
|
||||
self._started = False
|
||||
|
||||
try:
|
||||
await self._cleanup()
|
||||
|
||||
finally:
|
||||
# Remove transcripts and termination markers that may
|
||||
# have been queued before the abort completed.
|
||||
while True:
|
||||
try:
|
||||
self._events.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
|
||||
# Publish one termination marker for the event consumer.
|
||||
self._events.put_nowait(None)
|
||||
|
||||
|
||||
def events(self) -> AsyncIterator[TranscriptEvent]:
|
||||
return self._event_stream()
|
||||
|
||||
async def _event_stream(
|
||||
self,
|
||||
) -> AsyncIterator[TranscriptEvent]:
|
||||
while True:
|
||||
event = await self._events.get()
|
||||
|
||||
if event is None:
|
||||
break
|
||||
|
||||
yield event
|
||||
|
||||
async def _create_session(self) -> dict[str, Any]:
|
||||
if self._http is None:
|
||||
raise RuntimeError("HTTP client is not initialized.")
|
||||
|
||||
response = await self._http.post(
|
||||
f"http://{self.server}/v1/realtime/transcription_sessions"
|
||||
)
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
session = response.json()
|
||||
|
||||
if not isinstance(session, dict):
|
||||
raise RuntimeError(
|
||||
"NVIDIA NIM returned an invalid session response."
|
||||
)
|
||||
|
||||
return session
|
||||
|
||||
async def _connect(self) -> ClientConnection:
|
||||
return await websockets.connect(
|
||||
f"ws://{self.server}"
|
||||
"/v1/realtime"
|
||||
"?intent=transcription"
|
||||
)
|
||||
|
||||
async def _initialize_session(
|
||||
self,
|
||||
session: dict[str, Any],
|
||||
) -> None:
|
||||
if self._websocket is None:
|
||||
raise RuntimeError("NVIDIA ASR WebSocket is not connected.")
|
||||
|
||||
response = await self._websocket.recv()
|
||||
event = self._decode_event(response)
|
||||
|
||||
if event.get("type") != "conversation.created":
|
||||
raise RuntimeError(
|
||||
"Unexpected first NIM event: "
|
||||
f"{event.get('type')!r}"
|
||||
)
|
||||
|
||||
session["input_audio_format"] = "pcm16"
|
||||
|
||||
input_audio_transcription = dict(
|
||||
session.get("input_audio_transcription", {})
|
||||
)
|
||||
|
||||
input_audio_transcription["language"] = self.language_code
|
||||
session["input_audio_transcription"] = (
|
||||
input_audio_transcription
|
||||
)
|
||||
|
||||
session["input_audio_params"] = {
|
||||
"sample_rate_hz": CANONICAL_AUDIO_FORMAT.sample_rate,
|
||||
"num_channels": CANONICAL_AUDIO_FORMAT.channels,
|
||||
}
|
||||
|
||||
recognition_config = dict(
|
||||
session.get("recognition_config", {})
|
||||
)
|
||||
|
||||
recognition_config[
|
||||
"enable_automatic_punctuation"
|
||||
] = self.automatic_punctuation
|
||||
|
||||
session["recognition_config"] = recognition_config
|
||||
|
||||
await self._send_json(
|
||||
{
|
||||
"type": "transcription_session.update",
|
||||
"session": session,
|
||||
}
|
||||
)
|
||||
|
||||
response = await self._websocket.recv()
|
||||
event = self._decode_event(response)
|
||||
|
||||
if event.get("type") != "transcription_session.updated":
|
||||
raise RuntimeError(
|
||||
"NVIDIA NIM did not confirm session update: "
|
||||
f"{event.get('type')!r}"
|
||||
)
|
||||
|
||||
async def _receive_events(self) -> None:
|
||||
websocket = self._websocket
|
||||
|
||||
if websocket is None:
|
||||
return
|
||||
|
||||
try:
|
||||
async for raw_message in websocket:
|
||||
# Stop processing provider events after an explicit abort.
|
||||
if self._aborted:
|
||||
break
|
||||
event = self._decode_event(raw_message)
|
||||
event_type = event.get("type", "")
|
||||
|
||||
if (
|
||||
event_type
|
||||
== "conversation.item.input_audio_transcription.delta"
|
||||
):
|
||||
text = event.get("delta", "")
|
||||
|
||||
if text:
|
||||
await self._events.put(
|
||||
TranscriptEvent(
|
||||
type="partial",
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
|
||||
elif (
|
||||
event_type
|
||||
== "conversation.item.input_audio_transcription.completed"
|
||||
):
|
||||
text = event.get("transcript", "")
|
||||
|
||||
if not text:
|
||||
continue
|
||||
|
||||
is_last_result = event.get(
|
||||
"is_last_result",
|
||||
False,
|
||||
)
|
||||
|
||||
await self._events.put(
|
||||
TranscriptEvent(
|
||||
type=(
|
||||
"final"
|
||||
if is_last_result
|
||||
else "partial"
|
||||
),
|
||||
text=text,
|
||||
)
|
||||
)
|
||||
|
||||
if is_last_result:
|
||||
break
|
||||
|
||||
elif event_type.lower().startswith("error"):
|
||||
error = event.get("error", {})
|
||||
|
||||
raise RuntimeError(
|
||||
error.get(
|
||||
"message",
|
||||
"Unknown NVIDIA ASR error.",
|
||||
)
|
||||
)
|
||||
|
||||
except websockets.ConnectionClosed as exc:
|
||||
if not self._stopping:
|
||||
raise RuntimeError(
|
||||
"NVIDIA ASR WebSocket closed unexpectedly."
|
||||
) from exc
|
||||
|
||||
finally:
|
||||
if not self._aborted:
|
||||
await self._events.put(None)
|
||||
|
||||
async def _send_json(self, message: dict[str, Any]) -> None:
|
||||
if self._websocket is None:
|
||||
raise RuntimeError(
|
||||
"NVIDIA ASR WebSocket is not connected."
|
||||
)
|
||||
|
||||
await self._websocket.send(
|
||||
json.dumps(
|
||||
message,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _decode_event(
|
||||
message: str | bytes,
|
||||
) -> dict[str, Any]:
|
||||
if isinstance(message, bytes):
|
||||
message = message.decode("utf-8")
|
||||
|
||||
try:
|
||||
event = json.loads(message)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError(
|
||||
"Received invalid JSON from NVIDIA ASR."
|
||||
) from exc
|
||||
|
||||
if not isinstance(event, dict):
|
||||
raise RuntimeError(
|
||||
"Received invalid event from NVIDIA ASR."
|
||||
)
|
||||
|
||||
return event
|
||||
|
||||
async def _cleanup(self) -> None:
|
||||
"""
|
||||
Release NVIDIA transport resources.
|
||||
|
||||
Lifecycle flags are controlled by start_session(),
|
||||
stop_session(), and abort_session().
|
||||
"""
|
||||
self._started = False
|
||||
|
||||
current_task = asyncio.current_task()
|
||||
|
||||
if self._receiver_task is not None:
|
||||
if self._receiver_task is not current_task:
|
||||
if not self._receiver_task.done():
|
||||
self._receiver_task.cancel()
|
||||
|
||||
try:
|
||||
await self._receiver_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
self._receiver_task = None
|
||||
|
||||
if self._websocket is not None:
|
||||
try:
|
||||
await self._websocket.close()
|
||||
finally:
|
||||
self._websocket = None
|
||||
|
||||
if self._http is not None:
|
||||
await self._http.aclose()
|
||||
self._http = None
|
||||
9
src/voice_assistant/tts/__init__.py
Normal file
9
src/voice_assistant/tts/__init__.py
Normal file
@@ -0,0 +1,9 @@
|
||||
from .adapter import TTSAdapter
|
||||
from .mock import MockTTSAdapter
|
||||
from .voxcpm import VoxCPMTTSAdapter
|
||||
|
||||
__all__ = [
|
||||
"TTSAdapter",
|
||||
"MockTTSAdapter",
|
||||
"VoxCPMTTSAdapter",
|
||||
]
|
||||
12
src/voice_assistant/tts/adapter.py
Normal file
12
src/voice_assistant/tts/adapter.py
Normal file
@@ -0,0 +1,12 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
|
||||
class TTSAdapter(ABC):
|
||||
@abstractmethod
|
||||
async def synthesize(
|
||||
self,
|
||||
text: str,
|
||||
) -> AsyncIterator[bytes]:
|
||||
"""Convert text into audio chunks."""
|
||||
raise NotImplementedError
|
||||
18
src/voice_assistant/tts/mock.py
Normal file
18
src/voice_assistant/tts/mock.py
Normal file
@@ -0,0 +1,18 @@
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from .adapter import TTSAdapter
|
||||
|
||||
|
||||
class MockTTSAdapter(TTSAdapter):
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[str] = []
|
||||
|
||||
async def synthesize(
|
||||
self,
|
||||
text: str,
|
||||
) -> AsyncIterator[bytes]:
|
||||
self.calls.append(text)
|
||||
|
||||
yield b"\x01\x02\x03\x04"
|
||||
yield b"\x05\x06\x07\x08"
|
||||
|
||||
57
src/voice_assistant/tts/voxcpm.py
Normal file
57
src/voice_assistant/tts/voxcpm.py
Normal file
@@ -0,0 +1,57 @@
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import httpx
|
||||
|
||||
from .adapter import TTSAdapter
|
||||
|
||||
|
||||
class VoxCPMTTSAdapter(TTSAdapter):
|
||||
def __init__(
|
||||
self,
|
||||
server: str,
|
||||
model: str,
|
||||
voice: str,
|
||||
response_format: str = "pcm",
|
||||
stream: bool = True,
|
||||
stream_format: str = "audio",
|
||||
timeout: float = 120.0,
|
||||
) -> None:
|
||||
self.server = server.rstrip("/")
|
||||
self.model = model
|
||||
self.voice = voice
|
||||
self.response_format = response_format
|
||||
self.stream = stream
|
||||
self.stream_format = stream_format
|
||||
self.timeout = timeout
|
||||
|
||||
async def synthesize(
|
||||
self,
|
||||
text: str,
|
||||
) -> AsyncIterator[bytes]:
|
||||
if not text:
|
||||
raise ValueError("Text cannot be empty.")
|
||||
|
||||
payload = {
|
||||
"input": text,
|
||||
"model": self.model,
|
||||
"voice": self.voice,
|
||||
"response_format": self.response_format,
|
||||
"stream": self.stream,
|
||||
"stream_format": self.stream_format,
|
||||
}
|
||||
|
||||
url = f"{self.server}/v1/audio/speech"
|
||||
|
||||
timeout = httpx.Timeout(self.timeout)
|
||||
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
url,
|
||||
json=payload,
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
async for chunk in response.aiter_bytes():
|
||||
if chunk:
|
||||
yield chunk
|
||||
146
src/voice_assistant/turn_manager.py
Normal file
146
src/voice_assistant/turn_manager.py
Normal file
@@ -0,0 +1,146 @@
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class TurnState(StrEnum):
|
||||
LISTENING = "listening"
|
||||
PROCESSING = "processing"
|
||||
SPEAKING = "speaking"
|
||||
COMPLETED = "completed"
|
||||
INTERRUPTED = "interrupted"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Turn:
|
||||
turn_id: int
|
||||
state: TurnState = TurnState.LISTENING
|
||||
partial_text: str = ""
|
||||
final_text: str | None = None
|
||||
|
||||
|
||||
class TurnManager:
|
||||
def __init__(self) -> None:
|
||||
self._active_turn: Turn | None = None
|
||||
|
||||
@property
|
||||
def active_turn(self) -> Turn | None:
|
||||
return self._active_turn
|
||||
|
||||
def start(self, turn_id: int) -> Turn:
|
||||
if self._active_turn is not None:
|
||||
raise RuntimeError(
|
||||
"A turn is already active."
|
||||
)
|
||||
|
||||
self._active_turn = Turn(
|
||||
turn_id=turn_id,
|
||||
state=TurnState.LISTENING,
|
||||
)
|
||||
|
||||
return self._active_turn
|
||||
|
||||
def on_partial(
|
||||
self,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> None:
|
||||
turn = self._require_active_turn(turn_id)
|
||||
|
||||
if turn.state != TurnState.LISTENING:
|
||||
raise RuntimeError(
|
||||
f"Cannot receive partial transcript while "
|
||||
f"turn is {turn.state.value}."
|
||||
)
|
||||
|
||||
turn.partial_text = text
|
||||
|
||||
def on_final(
|
||||
self,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> None:
|
||||
turn = self._require_active_turn(turn_id)
|
||||
|
||||
if turn.state != TurnState.LISTENING:
|
||||
raise RuntimeError(
|
||||
f"Cannot receive final transcript while "
|
||||
f"turn is {turn.state.value}."
|
||||
)
|
||||
|
||||
turn.final_text = text
|
||||
turn.partial_text = text
|
||||
turn.state = TurnState.PROCESSING
|
||||
|
||||
def on_agent_response(
|
||||
self,
|
||||
turn_id: int,
|
||||
) -> None:
|
||||
turn = self._require_active_turn(turn_id)
|
||||
|
||||
if turn.state != TurnState.PROCESSING:
|
||||
raise RuntimeError(
|
||||
f"Cannot start speaking while "
|
||||
f"turn is {turn.state.value}."
|
||||
)
|
||||
|
||||
turn.state = TurnState.SPEAKING
|
||||
|
||||
def on_tts_complete(
|
||||
self,
|
||||
turn_id: int,
|
||||
) -> None:
|
||||
turn = self._require_active_turn(turn_id)
|
||||
|
||||
if turn.state != TurnState.SPEAKING:
|
||||
raise RuntimeError(
|
||||
f"Cannot complete TTS while "
|
||||
f"turn is {turn.state.value}."
|
||||
)
|
||||
|
||||
turn.state = TurnState.COMPLETED
|
||||
|
||||
def stop(self, turn_id: int) -> Turn:
|
||||
"""
|
||||
Return the active turn when its complete lifecycle has finished.
|
||||
|
||||
A turn may only be stopped after the TTS stage has completed and
|
||||
the semantic state has reached COMPLETED.
|
||||
"""
|
||||
turn = self._require_active_turn(turn_id)
|
||||
|
||||
if turn.state != TurnState.COMPLETED:
|
||||
raise RuntimeError(
|
||||
f"Cannot stop turn while turn is {turn.state.value}."
|
||||
)
|
||||
|
||||
return turn
|
||||
|
||||
def interrupt(self, turn_id: int) -> Turn:
|
||||
turn = self._require_active_turn(turn_id)
|
||||
|
||||
turn.state = TurnState.INTERRUPTED
|
||||
return turn
|
||||
|
||||
def clear(self, turn_id: int) -> None:
|
||||
"""
|
||||
Clear the active turn after its lifecycle has been handled.
|
||||
|
||||
Turn identity validation is delegated to _require_active_turn().
|
||||
"""
|
||||
self._require_active_turn(turn_id)
|
||||
self._active_turn = None
|
||||
|
||||
|
||||
def _require_active_turn(self, turn_id: int) -> Turn:
|
||||
if self._active_turn is None:
|
||||
raise RuntimeError(
|
||||
"There is no active turn."
|
||||
)
|
||||
|
||||
if self._active_turn.turn_id != turn_id:
|
||||
raise ValueError(
|
||||
f"Unexpected turn_id={turn_id}; "
|
||||
f"active_turn_id={self._active_turn.turn_id}."
|
||||
)
|
||||
|
||||
return self._active_turn
|
||||
476
src/voice_assistant/voice_pipeline.py
Normal file
476
src/voice_assistant/voice_pipeline.py
Normal file
@@ -0,0 +1,476 @@
|
||||
"""
|
||||
Voice turn lifecycle and provider orchestration for the Voice Assistant.
|
||||
|
||||
Purpose:
|
||||
Coordinate session and semantic turn lifecycle and orchestrate
|
||||
provider-independent Agent and TTS processing.
|
||||
|
||||
Responsibilities:
|
||||
- Start, complete, and interrupt turns.
|
||||
- Keep SessionState and TurnManager synchronized.
|
||||
- Send final transcript text to the configured Agent.
|
||||
- Convert the Agent response into streaming TTS audio.
|
||||
- Expose provider-independent results to the transport layer.
|
||||
|
||||
Non-responsibilities:
|
||||
- Managing WebSocket or FastAPI connections.
|
||||
- Parsing WebSocket protocol messages.
|
||||
- Managing provider-specific implementation details.
|
||||
- Loading configuration or selecting providers.
|
||||
- Sending protocol events to clients.
|
||||
|
||||
Architecture:
|
||||
Audio Gateway
|
||||
|
|
||||
v
|
||||
VoicePipeline
|
||||
| |
|
||||
v v
|
||||
Session TurnManager
|
||||
|
|
||||
+---- AgentAdapter
|
||||
|
|
||||
+---- TTSAdapter
|
||||
|
||||
The pipeline coordinates the Voice Core while remaining independent
|
||||
from WebSocket transport and concrete provider implementations.
|
||||
"""
|
||||
|
||||
|
||||
import asyncio
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
from voice_assistant.agent.adapter import AgentAdapter
|
||||
from voice_assistant.audio_gateway.app.session import SessionState
|
||||
from voice_assistant.turn_manager import Turn, TurnManager
|
||||
from voice_assistant.tts.adapter import TTSAdapter
|
||||
|
||||
|
||||
DEFAULT_TURN_CANCEL_TIMEOUT = 5.0
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PipelineResult:
|
||||
"""
|
||||
Result produced after processing a final transcript.
|
||||
|
||||
The result contains only provider-independent data. Audio streaming
|
||||
remains a separate operation so the transport layer can consume it
|
||||
without coupling the result object to an AsyncIterator lifecycle.
|
||||
"""
|
||||
|
||||
response_text: str
|
||||
|
||||
|
||||
class VoicePipeline:
|
||||
"""
|
||||
Coordinate session, turn, Agent, and TTS processing.
|
||||
|
||||
Lifecycle operations are synchronous because they only update local
|
||||
state. Provider execution remains asynchronous because Agent and TTS
|
||||
may involve network or streaming operations.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session: SessionState,
|
||||
turn_manager: TurnManager,
|
||||
agent: AgentAdapter,
|
||||
tts: TTSAdapter,
|
||||
) -> None:
|
||||
self.session = session
|
||||
self.turn_manager = turn_manager
|
||||
self.agent = agent
|
||||
self.tts = tts
|
||||
self._turn_task: asyncio.Task[None] | None = None
|
||||
|
||||
|
||||
def start_turn(self) -> Turn:
|
||||
"""
|
||||
Start a new turn only when no previous execution is active.
|
||||
|
||||
Purpose:
|
||||
Prevent overlapping voice turns after interruption
|
||||
or cancellation timeout.
|
||||
"""
|
||||
if (
|
||||
self._turn_task is not None
|
||||
and not self._turn_task.done()
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Cannot start a new turn while the previous "
|
||||
"turn task is still running."
|
||||
)
|
||||
|
||||
turn_id = self.session.start_turn()
|
||||
|
||||
try:
|
||||
return self.turn_manager.start(turn_id)
|
||||
except Exception:
|
||||
self.session.interrupt(turn_id)
|
||||
raise
|
||||
|
||||
async def process_transcript(
|
||||
self,
|
||||
session_id: str,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> PipelineResult:
|
||||
"""
|
||||
Process final transcript text through the configured Agent.
|
||||
|
||||
The turn must be in PROCESSING state before the Agent is invoked.
|
||||
Once a response is produced, the semantic turn transitions to
|
||||
SPEAKING so that TTS can be executed.
|
||||
"""
|
||||
turn = self.turn_manager.active_turn
|
||||
|
||||
if (
|
||||
turn is None
|
||||
or turn.turn_id != turn_id
|
||||
):
|
||||
raise ValueError(
|
||||
f"Unexpected turn_id={turn_id}."
|
||||
)
|
||||
|
||||
if turn.state.name != "PROCESSING":
|
||||
raise RuntimeError(
|
||||
f"Cannot process transcript while turn is "
|
||||
f"{turn.state.value}."
|
||||
)
|
||||
|
||||
response_text = await self.agent.process(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=text,
|
||||
)
|
||||
|
||||
self.turn_manager.on_agent_response(turn_id)
|
||||
|
||||
return PipelineResult(
|
||||
response_text=response_text,
|
||||
)
|
||||
|
||||
async def synthesize_response(
|
||||
self,
|
||||
response_text: str,
|
||||
) -> AsyncIterator[bytes]:
|
||||
"""
|
||||
Stream TTS audio for an Agent response.
|
||||
|
||||
The TTS adapter remains responsible for provider-specific
|
||||
synthesis and audio streaming behavior.
|
||||
"""
|
||||
async for audio_chunk in self.tts.synthesize(
|
||||
response_text,
|
||||
):
|
||||
yield audio_chunk
|
||||
|
||||
def complete_turn(
|
||||
self,
|
||||
turn_id: int,
|
||||
) -> Turn:
|
||||
"""
|
||||
Complete and release a successfully processed turn.
|
||||
|
||||
The TurnManager must already have reached COMPLETED through
|
||||
successful TTS completion.
|
||||
"""
|
||||
turn = self.turn_manager.stop(turn_id)
|
||||
|
||||
self.turn_manager.clear(turn_id)
|
||||
self.session.end_turn(turn_id)
|
||||
|
||||
return turn
|
||||
|
||||
def interrupt_turn(
|
||||
self,
|
||||
turn_id: int,
|
||||
) -> Turn:
|
||||
"""
|
||||
Interrupt and release the active turn.
|
||||
|
||||
Actual cancellation of asynchronous provider tasks will be
|
||||
added in the lifecycle/cancellation phase.
|
||||
"""
|
||||
turn = self.turn_manager.interrupt(turn_id)
|
||||
|
||||
self.session.interrupt(turn_id)
|
||||
self.turn_manager.clear(turn_id)
|
||||
|
||||
return turn
|
||||
|
||||
|
||||
def on_partial(
|
||||
self,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> None:
|
||||
"""Apply a partial STT transcript to the active turn."""
|
||||
self.turn_manager.on_partial(
|
||||
turn_id,
|
||||
text,
|
||||
)
|
||||
|
||||
|
||||
def on_final(
|
||||
self,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> None:
|
||||
"""Apply a final STT transcript to the active turn."""
|
||||
self.turn_manager.on_final(
|
||||
turn_id,
|
||||
text,
|
||||
)
|
||||
|
||||
|
||||
def on_tts_complete(
|
||||
self,
|
||||
turn_id: int,
|
||||
) -> None:
|
||||
"""Mark TTS completion for the active turn."""
|
||||
self.turn_manager.on_tts_complete(
|
||||
turn_id,
|
||||
)
|
||||
|
||||
|
||||
def on_agent_response(
|
||||
self,
|
||||
turn_id: int,
|
||||
) -> None:
|
||||
"""Mark the Agent stage as complete and start TTS."""
|
||||
self.turn_manager.on_agent_response(
|
||||
turn_id,
|
||||
)
|
||||
|
||||
|
||||
def close_session(self) -> None:
|
||||
"""
|
||||
Close the voice session and release its active turn state.
|
||||
|
||||
Purpose:
|
||||
Ensure SessionState and TurnManager remain consistent
|
||||
when the session ends, even if a turn is still active.
|
||||
|
||||
Responsibilities:
|
||||
- Mark an active semantic turn as interrupted.
|
||||
- Release the active turn from TurnManager.
|
||||
- Close SessionState.
|
||||
|
||||
Non-responsibilities:
|
||||
- Cancelling asynchronous provider tasks.
|
||||
- Closing provider network connections.
|
||||
- Sending WebSocket protocol events.
|
||||
"""
|
||||
active_turn = self.turn_manager.active_turn
|
||||
|
||||
if active_turn is not None:
|
||||
self.turn_manager.interrupt(active_turn.turn_id)
|
||||
self.turn_manager.clear(active_turn.turn_id)
|
||||
|
||||
self.session.close()
|
||||
|
||||
|
||||
def is_active_turn(self, turn_id: int) -> bool:
|
||||
"""
|
||||
Check whether the requested turn is currently active.
|
||||
|
||||
Purpose:
|
||||
Validate turn ownership without exposing TurnManager
|
||||
internals to the Gateway.
|
||||
|
||||
Responsibilities:
|
||||
- Verify the active semantic turn ID.
|
||||
- Verify the active session turn ID.
|
||||
- Return a boolean without changing state.
|
||||
|
||||
Non-responsibilities:
|
||||
- Interrupting turns.
|
||||
- Cancelling asynchronous tasks.
|
||||
- Closing provider connections.
|
||||
"""
|
||||
active_turn = self.turn_manager.active_turn
|
||||
|
||||
return (
|
||||
active_turn is not None
|
||||
and active_turn.turn_id == turn_id
|
||||
and self.session.active_turn_id == turn_id
|
||||
)
|
||||
|
||||
|
||||
def interrupt_if_active(
|
||||
self,
|
||||
turn_id: int,
|
||||
) -> bool:
|
||||
"""
|
||||
Interrupt a turn only when it matches the active semantic turn.
|
||||
|
||||
Purpose:
|
||||
Preserve the Gateway's current tolerant interruption behavior
|
||||
without exposing TurnManager internals.
|
||||
|
||||
Returns:
|
||||
True if the matching turn was interrupted.
|
||||
False if no matching semantic turn exists.
|
||||
|
||||
Non-responsibilities:
|
||||
- Cancelling asynchronous provider tasks.
|
||||
- Closing provider connections.
|
||||
- Sending WebSocket events.
|
||||
"""
|
||||
active_turn = self.turn_manager.active_turn
|
||||
|
||||
if (
|
||||
active_turn is None
|
||||
or active_turn.turn_id != turn_id
|
||||
):
|
||||
# Preserve the previous Gateway behavior when no matching
|
||||
# semantic turn exists.
|
||||
self.session.interrupt(turn_id)
|
||||
return False
|
||||
|
||||
self.interrupt_turn(turn_id)
|
||||
return True
|
||||
|
||||
|
||||
def register_turn_task(
|
||||
self,
|
||||
task: asyncio.Task[None],
|
||||
) -> None:
|
||||
"""
|
||||
Register the asynchronous execution task for the active turn.
|
||||
|
||||
Responsibilities:
|
||||
- Associate one task with the active semantic turn.
|
||||
- Reject duplicate running tasks.
|
||||
- Preserve task ownership inside the VoicePipeline.
|
||||
|
||||
Non-responsibilities:
|
||||
- Creating tasks.
|
||||
- Cancelling tasks.
|
||||
- Awaiting task completion.
|
||||
"""
|
||||
if self.turn_manager.active_turn is None:
|
||||
raise RuntimeError(
|
||||
"Cannot register task without an active turn."
|
||||
)
|
||||
|
||||
if self._turn_task is not None and not self._turn_task.done():
|
||||
raise RuntimeError(
|
||||
"A turn execution task is already active."
|
||||
)
|
||||
|
||||
self._turn_task = task
|
||||
|
||||
|
||||
@property
|
||||
def turn_task(self) -> asyncio.Task[None] | None:
|
||||
"""Return the currently registered turn execution task."""
|
||||
return self._turn_task
|
||||
|
||||
|
||||
async def wait_turn_task(self) -> None:
|
||||
"""
|
||||
Await normal completion of the registered turn execution task.
|
||||
|
||||
Purpose:
|
||||
Release task ownership after successful or failed execution
|
||||
without requesting cancellation.
|
||||
|
||||
Responsibilities:
|
||||
- Await the registered task.
|
||||
- Propagate execution failures.
|
||||
- Preserve cancellation of the waiting operation.
|
||||
- Clear ownership only after task termination.
|
||||
|
||||
Non-responsibilities:
|
||||
- Cancelling the task.
|
||||
- Updating semantic turn state.
|
||||
- Closing STT provider connections.
|
||||
"""
|
||||
task = self._turn_task
|
||||
|
||||
if task is None:
|
||||
return
|
||||
|
||||
if task is asyncio.current_task():
|
||||
raise RuntimeError(
|
||||
"A turn task cannot await itself."
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.shield(task)
|
||||
|
||||
finally:
|
||||
if task.done() and self._turn_task is task:
|
||||
self._turn_task = None
|
||||
|
||||
|
||||
async def cancel_turn_task(
|
||||
self,
|
||||
timeout: float = DEFAULT_TURN_CANCEL_TIMEOUT,
|
||||
) -> None:
|
||||
"""
|
||||
Cancel and await the registered turn execution task.
|
||||
|
||||
Purpose:
|
||||
Bound cancellation latency without losing ownership of
|
||||
a task that has not terminated.
|
||||
|
||||
Responsibilities:
|
||||
- Reject self-cancellation.
|
||||
- Request cancellation of the active task.
|
||||
- Wait for termination within a configurable timeout.
|
||||
- Preserve ownership if cancellation times out.
|
||||
- Propagate unexpected task failures.
|
||||
- Release ownership only after task termination.
|
||||
|
||||
Non-responsibilities:
|
||||
- Updating semantic turn state.
|
||||
- Closing STT provider connections.
|
||||
- Sending WebSocket events.
|
||||
"""
|
||||
if timeout <= 0:
|
||||
raise ValueError(
|
||||
"Turn cancellation timeout must be positive."
|
||||
)
|
||||
|
||||
task = self._turn_task
|
||||
|
||||
if task is None:
|
||||
return
|
||||
|
||||
if task is asyncio.current_task():
|
||||
raise RuntimeError(
|
||||
"A turn task cannot cancel itself."
|
||||
)
|
||||
|
||||
try:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
try:
|
||||
async with asyncio.timeout(timeout):
|
||||
await asyncio.shield(task)
|
||||
|
||||
except TimeoutError as exc:
|
||||
raise TimeoutError(
|
||||
"Turn task cancellation timed out."
|
||||
) from exc
|
||||
|
||||
except asyncio.CancelledError:
|
||||
current = asyncio.current_task()
|
||||
|
||||
if current is not None and current.cancelling():
|
||||
raise
|
||||
|
||||
if not task.cancelled():
|
||||
raise
|
||||
|
||||
finally:
|
||||
if task.done() and self._turn_task is task:
|
||||
self._turn_task = None
|
||||
|
||||
40
tests/test_agent.py
Normal file
40
tests/test_agent.py
Normal file
@@ -0,0 +1,40 @@
|
||||
import pytest
|
||||
|
||||
from voice_assistant.agent import MockAgentAdapter
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def text_mock_agent_process_text():
|
||||
agent = MackAgentAdapter()
|
||||
|
||||
response = await agent.process(
|
||||
session_id="session-1",
|
||||
turn_id=1,
|
||||
text="Qual o horário de funcionamento?",
|
||||
)
|
||||
|
||||
assert response == (
|
||||
"Resposta simulada para: "
|
||||
"Qual o horário de funcionamento?"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mock_agent_records_call():
|
||||
agent = MockAgentAdapter()
|
||||
|
||||
await agent.process(
|
||||
session_id="session-1",
|
||||
turn_id=7,
|
||||
text= "Olá",
|
||||
)
|
||||
|
||||
|
||||
assert agent.calls == [
|
||||
(
|
||||
"session-1",
|
||||
7,
|
||||
"Olá",
|
||||
)
|
||||
]
|
||||
|
||||
31
tests/test_audio_models.py
Normal file
31
tests/test_audio_models.py
Normal file
@@ -0,0 +1,31 @@
|
||||
"""
|
||||
Tests for the canonical Voice Assistant audio contract.
|
||||
|
||||
Purpose:
|
||||
Verify that the Voice Core exposes the expected provider-independent
|
||||
canonical audio format.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from voice_assistant.audio.models import (
|
||||
AudioFormat,
|
||||
CANONICAL_AUDIO_FORMAT,
|
||||
)
|
||||
|
||||
|
||||
def test_canonical_audio_format():
|
||||
assert CANONICAL_AUDIO_FORMAT.encoding == "pcm_s16le"
|
||||
assert CANONICAL_AUDIO_FORMAT.sample_rate == 16000
|
||||
assert CANONICAL_AUDIO_FORMAT.channels == 1
|
||||
|
||||
|
||||
def test_audio_format_rejects_unsupported_encoding():
|
||||
with pytest.raises(ValidationError):
|
||||
AudioFormat(
|
||||
encoding="mp3",
|
||||
sample_rate=16000,
|
||||
channels=1,
|
||||
)
|
||||
|
||||
288
tests/test_config.py
Normal file
288
tests/test_config.py
Normal file
@@ -0,0 +1,288 @@
|
||||
"""
|
||||
Typed configuration models for the Voice Assistant.
|
||||
|
||||
Purpose:
|
||||
Define and validate the configuration structures used by the
|
||||
application and dynamic provider loading system.
|
||||
|
||||
Responsibilities:
|
||||
- Define the application configuration for STT, Agent, and TTS.
|
||||
- Define the provider catalog structure.
|
||||
- Represent provider-specific options without coupling the core
|
||||
configuration model to a specific vendor or implementation.
|
||||
- Describe provider capabilities for discovery and future tooling.
|
||||
|
||||
Non-responsibilities:
|
||||
- Loading YAML files.
|
||||
- Reading environment variables or secrets.
|
||||
- Selecting or instantiating provider implementations.
|
||||
- Executing STT, Agent, or TTS functionality.
|
||||
|
||||
Architecture:
|
||||
YAML
|
||||
|
|
||||
v
|
||||
Config Loader
|
||||
|
|
||||
v
|
||||
AppConfig / ProviderCatalog
|
||||
|
|
||||
v
|
||||
Provider Loader
|
||||
|
|
||||
v
|
||||
STTAdapter / AgentAdapter / TTSAdapter
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from voice_assistant.config.loader import load_config
|
||||
from voice_assistant.config.loader import (
|
||||
load_config,
|
||||
load_provider_catalog,
|
||||
)
|
||||
|
||||
|
||||
def test_load_development_config():
|
||||
config = load_config()
|
||||
|
||||
assert config.stt.provider == "nvidia"
|
||||
assert config.stt.options["server"] == "127.0.0.1:9000"
|
||||
assert config.stt.options["language_code"] == "pt-BR"
|
||||
assert config.stt.options["automatic_punctuation"] is True
|
||||
|
||||
assert config.tts.provider == "voxcpm"
|
||||
assert config.tts.options["server"] == "http://127.0.0.1:8000"
|
||||
assert config.tts.options["model"] == "openbmb/VoxCPM2"
|
||||
assert config.tts.options["voice"] == "default"
|
||||
|
||||
assert config.agent.provider == "mock"
|
||||
|
||||
|
||||
def test_load_config_missing_file(tmp_path):
|
||||
with pytest.raises(
|
||||
FileNotFoundError,
|
||||
match="Configuration file not found",
|
||||
):
|
||||
load_config(
|
||||
environment="development",
|
||||
config_dir=tmp_path,
|
||||
)
|
||||
|
||||
|
||||
def test_load_config_empty_file(tmp_path):
|
||||
config_file = tmp_path / "development.yaml"
|
||||
config_file.write_text(
|
||||
"",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Configuration file is empty",
|
||||
):
|
||||
load_config(
|
||||
environment="development",
|
||||
config_dir=tmp_path,
|
||||
)
|
||||
|
||||
|
||||
def test_load_config_root_must_be_mapping(tmp_path):
|
||||
config_file = tmp_path / "development.yaml"
|
||||
config_file.write_text(
|
||||
"- item1\n- item2\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Configuration root must be a mapping",
|
||||
):
|
||||
load_config(
|
||||
environment="development",
|
||||
config_dir=tmp_path,
|
||||
)
|
||||
|
||||
|
||||
def test_load_config_validates_required_fields(tmp_path):
|
||||
config_file = tmp_path / "development.yaml"
|
||||
|
||||
config_file.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"stt": {
|
||||
"provider": "nvidia",
|
||||
"server": "127.0.0.1:9000",
|
||||
"language": "pt-BR",
|
||||
},
|
||||
"tts": {
|
||||
"provider": "voxcpm",
|
||||
"server": "http://127.0.0.1:8000",
|
||||
"model": "openbmb/VoxCPM2",
|
||||
"voice": "default",
|
||||
},
|
||||
# agent intentionally missing
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
Exception,
|
||||
match="agent",
|
||||
):
|
||||
load_config(
|
||||
environment="development",
|
||||
config_dir=tmp_path,
|
||||
)
|
||||
|
||||
def test_load_provider_catalog():
|
||||
catalog = load_provider_catalog()
|
||||
|
||||
assert "nvidia" in catalog.providers["stt"]
|
||||
assert "voxcpm" in catalog.providers["tts"]
|
||||
assert "mock" in catalog.providers["agent"]
|
||||
|
||||
voxcpm = catalog.providers["tts"]["voxcpm"]
|
||||
|
||||
assert voxcpm.name == "VoxCPM"
|
||||
assert (
|
||||
voxcpm.implementation
|
||||
== "voice_assistant.tts.voxcpm:VoxCPMTTSAdapter"
|
||||
)
|
||||
assert voxcpm.capabilities == {
|
||||
"realtime": True,
|
||||
"streaming": True,
|
||||
}
|
||||
|
||||
|
||||
def test_load_provider_catalog_missing_file(tmp_path):
|
||||
with pytest.raises(
|
||||
FileNotFoundError,
|
||||
match="Provider catalog not found",
|
||||
):
|
||||
load_provider_catalog(
|
||||
config_dir=tmp_path,
|
||||
)
|
||||
|
||||
|
||||
def test_load_config_uses_app_env(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
development = tmp_path / "development.yaml"
|
||||
|
||||
development.write_text(
|
||||
"""
|
||||
stt:
|
||||
provider: nvidia
|
||||
options:
|
||||
server: "dev-asr"
|
||||
|
||||
tts:
|
||||
provider: voxcpm
|
||||
options:
|
||||
server: "dev-tts"
|
||||
model: "dev-model"
|
||||
voice: "default"
|
||||
|
||||
agent:
|
||||
provider: mock
|
||||
options: {}
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setenv(
|
||||
"APP_ENV",
|
||||
"development",
|
||||
)
|
||||
|
||||
config = load_config(
|
||||
config_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert config.stt.options["server"] == "dev-asr"
|
||||
assert config.tts.options["server"] == "dev-tts"
|
||||
assert config.tts.options["model"] == "dev-model"
|
||||
|
||||
|
||||
def test_load_config_rejects_unknown_environment(
|
||||
tmp_path,
|
||||
):
|
||||
with pytest.raises(
|
||||
FileNotFoundError,
|
||||
match="Configuration file not found",
|
||||
):
|
||||
load_config(
|
||||
environment="does-not-exist",
|
||||
config_dir=tmp_path,
|
||||
)
|
||||
|
||||
def test_load_config_uses_app_env(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
config_file = tmp_path / "staging.yaml"
|
||||
|
||||
config_file.write_text(
|
||||
"""
|
||||
stt:
|
||||
provider: nvidia
|
||||
options:
|
||||
server: "staging-asr:9000"
|
||||
|
||||
tts:
|
||||
provider: voxcpm
|
||||
options:
|
||||
server: "http://staging-tts:8000"
|
||||
model: "staging-model"
|
||||
voice: "staging-voice"
|
||||
|
||||
agent:
|
||||
provider: mock
|
||||
options: {}
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setenv(
|
||||
"APP_ENV",
|
||||
"staging",
|
||||
)
|
||||
|
||||
config = load_config(
|
||||
config_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert config.stt.provider == "nvidia"
|
||||
assert (
|
||||
config.stt.options["server"]
|
||||
== "staging-asr:9000"
|
||||
)
|
||||
|
||||
assert config.tts.provider == "voxcpm"
|
||||
assert (
|
||||
config.tts.options["server"]
|
||||
== "http://staging-tts:8000"
|
||||
)
|
||||
|
||||
assert config.tts.options["model"] == "staging-model"
|
||||
assert config.tts.options["voice"] == "staging-voice"
|
||||
|
||||
assert config.agent.provider == "mock"
|
||||
|
||||
|
||||
def test_load_config_rejects_unknown_environment(
|
||||
tmp_path,
|
||||
):
|
||||
with pytest.raises(
|
||||
FileNotFoundError,
|
||||
match="Configuration file not found",
|
||||
):
|
||||
load_config(
|
||||
environment="does-not-exist",
|
||||
config_dir=tmp_path,
|
||||
)
|
||||
|
||||
1701
tests/test_gateway.py
Normal file
1701
tests/test_gateway.py
Normal file
File diff suppressed because it is too large
Load Diff
0
tests/test_nvidia_stt.py
Normal file
0
tests/test_nvidia_stt.py
Normal file
71
tests/test_nvidia_stt_integration.py
Normal file
71
tests/test_nvidia_stt_integration.py
Normal file
@@ -0,0 +1,71 @@
|
||||
import asyncio
|
||||
import wave
|
||||
|
||||
from voice_assistant.stt.nvidia import NvidiaSTTAdapter
|
||||
|
||||
|
||||
AUDIO_FILE = (
|
||||
"/home/ubuntu/voice-assistant/audios/chafik_normalize.wav"
|
||||
)
|
||||
|
||||
CHUNK_FRAMES = 1600
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
stt = NvidiaSTTAdapter(
|
||||
server="127.0.0.1:9000",
|
||||
language_code="pt-BR",
|
||||
automatic_punctuation=True,
|
||||
)
|
||||
|
||||
print("Starting STT session...")
|
||||
|
||||
await stt.start_session()
|
||||
|
||||
print("STT session ready.")
|
||||
|
||||
async def send_audio() -> None:
|
||||
with wave.open(AUDIO_FILE, "rb") as wav:
|
||||
while True:
|
||||
audio = wav.readframes(CHUNK_FRAMES)
|
||||
|
||||
if not audio:
|
||||
break
|
||||
|
||||
await stt.send_audio(audio)
|
||||
|
||||
async def receive_events() -> None:
|
||||
async for event in stt.events():
|
||||
print(
|
||||
f"[{event.type.upper():7}] {event.text}"
|
||||
)
|
||||
|
||||
sender = asyncio.create_task(send_audio())
|
||||
receiver = asyncio.create_task(receive_events())
|
||||
|
||||
try:
|
||||
await sender
|
||||
print("Audio stream sent.")
|
||||
|
||||
await stt.stop_session()
|
||||
|
||||
await receiver
|
||||
|
||||
finally:
|
||||
if not sender.done():
|
||||
sender.cancel()
|
||||
|
||||
if not receiver.done():
|
||||
receiver.cancel()
|
||||
|
||||
await asyncio.gather(
|
||||
sender,
|
||||
receiver,
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
print("STT session finished.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
170
tests/test_protocol.py
Normal file
170
tests/test_protocol.py
Normal file
@@ -0,0 +1,170 @@
|
||||
"""
|
||||
Tests for the Voice Assistant WebSocket protocol.
|
||||
|
||||
Purpose:
|
||||
Verify that protocol message models enforce their declared
|
||||
event types and canonical audio contract.
|
||||
|
||||
Responsibilities:
|
||||
- Validate inbound protocol event discriminators.
|
||||
- Validate SessionStart audio configuration.
|
||||
- Reject messages whose type does not match their model.
|
||||
|
||||
Non-responsibilities:
|
||||
- Testing WebSocket transport behavior.
|
||||
- Testing session or turn lifecycle.
|
||||
- Testing STT, Agent, or TTS providers.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
|
||||
from voice_assistant.audio_gateway.app.protocol import (
|
||||
AgentResponse,
|
||||
AudioChunk,
|
||||
AudioStart,
|
||||
AudioStop,
|
||||
ErrorEvent,
|
||||
Interrupt,
|
||||
Interrupted,
|
||||
SessionEnd,
|
||||
SessionEnded,
|
||||
SessionReady,
|
||||
SessionStart,
|
||||
STTFinal,
|
||||
STTPartial,
|
||||
TurnEnded,
|
||||
TurnStarted,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected_type"),
|
||||
[
|
||||
(SessionStart, "session.start"),
|
||||
(AudioStart, "audio.start"),
|
||||
(AudioChunk, "audio.chunk"),
|
||||
(AudioStop, "audio.stop"),
|
||||
(Interrupt, "interrupt"),
|
||||
(SessionEnd, "session.end"),
|
||||
],
|
||||
)
|
||||
def test_protocol_event_has_expected_type(
|
||||
model,
|
||||
expected_type,
|
||||
):
|
||||
payload = {
|
||||
"type": expected_type,
|
||||
}
|
||||
|
||||
if model in (AudioChunk, AudioStop, Interrupt):
|
||||
payload["turn_id"] = 1
|
||||
|
||||
if model is AudioChunk:
|
||||
payload["sequence_number"] = 0
|
||||
|
||||
event = model.model_validate(payload)
|
||||
|
||||
assert event.type == expected_type
|
||||
|
||||
|
||||
def test_protocol_rejects_wrong_event_type():
|
||||
with pytest.raises(ValidationError):
|
||||
AudioStart.model_validate(
|
||||
{
|
||||
"type": "audio.stop",
|
||||
}
|
||||
)
|
||||
|
||||
def test_session_ready_output():
|
||||
event = SessionReady(
|
||||
protocol_version="1.0",
|
||||
session_id="session-1",
|
||||
audio=CANONICAL_AUDIO_FORMAT,
|
||||
)
|
||||
|
||||
assert event.model_dump() == {
|
||||
"type": "session.ready",
|
||||
"protocol_version": "1.0",
|
||||
"session_id": "session-1",
|
||||
"audio": {
|
||||
"encoding": "pcm_s16le",
|
||||
"sample_rate": 16000,
|
||||
"channels": 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "event_type"),
|
||||
[
|
||||
(TurnStarted, "turn.started"),
|
||||
(STTPartial, "stt.partial"),
|
||||
(STTFinal, "stt.final"),
|
||||
(AgentResponse, "agent.response"),
|
||||
(TurnEnded, "turn.ended"),
|
||||
(Interrupted, "interrupted"),
|
||||
],
|
||||
)
|
||||
def test_turn_output_event_types(
|
||||
model,
|
||||
event_type,
|
||||
):
|
||||
data = {
|
||||
"session_id": "session-1",
|
||||
"turn_id": 1,
|
||||
}
|
||||
|
||||
if model in (STTPartial, STTFinal, AgentResponse):
|
||||
data["text"] = "test"
|
||||
|
||||
event = model(**data)
|
||||
|
||||
assert event.type == event_type
|
||||
|
||||
|
||||
def test_session_ended_output():
|
||||
event = SessionEnded(
|
||||
session_id="session-1",
|
||||
)
|
||||
|
||||
assert event.type == "session.ended"
|
||||
assert event.session_id == "session-1"
|
||||
|
||||
def test_error_event():
|
||||
event = ErrorEvent(
|
||||
session_id="session-1",
|
||||
turn_id=1,
|
||||
code="TEST_ERROR",
|
||||
message="Something went wrong.",
|
||||
retryable=True,
|
||||
details={
|
||||
"reason": "test",
|
||||
},
|
||||
)
|
||||
|
||||
assert event.model_dump() == {
|
||||
"type": "error",
|
||||
"session_id": "session-1",
|
||||
"turn_id": 1,
|
||||
"code": "TEST_ERROR",
|
||||
"message": "Something went wrong.",
|
||||
"retryable": True,
|
||||
"details": {
|
||||
"reason": "test",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_error_event_defaults():
|
||||
event = ErrorEvent(
|
||||
code="TEST_ERROR",
|
||||
message="Something went wrong.",
|
||||
)
|
||||
|
||||
assert event.type == "error"
|
||||
assert event.session_id is None
|
||||
assert event.turn_id is None
|
||||
assert event.retryable is False
|
||||
assert event.details == {}
|
||||
|
||||
83
tests/test_provider_factories.py
Normal file
83
tests/test_provider_factories.py
Normal file
@@ -0,0 +1,83 @@
|
||||
"""
|
||||
Provider factory tests for the Voice Assistant.
|
||||
|
||||
Purpose:
|
||||
Verify that the provider composition layer resolves the configured
|
||||
providers dynamically and passes the configured options correctly.
|
||||
|
||||
Responsibilities:
|
||||
- Validate STT provider creation.
|
||||
- Validate Agent provider creation.
|
||||
- Validate TTS provider creation.
|
||||
- Verify provider-specific options are propagated to each adapter.
|
||||
|
||||
Non-responsibilities:
|
||||
- Testing provider internals.
|
||||
- Testing WebSocket behavior.
|
||||
- Testing external services.
|
||||
|
||||
Architecture:
|
||||
AppConfig + ProviderCatalog
|
||||
|
|
||||
v
|
||||
provider factories
|
||||
|
|
||||
v
|
||||
concrete Adapter instances
|
||||
"""
|
||||
|
||||
from voice_assistant.agent.mock import MockAgentAdapter
|
||||
from voice_assistant.config.loader import (
|
||||
load_config,
|
||||
load_provider_catalog,
|
||||
)
|
||||
from voice_assistant.providers import (
|
||||
create_agent_adapter,
|
||||
create_stt_adapter,
|
||||
create_tts_adapter,
|
||||
)
|
||||
from voice_assistant.stt.nvidia import NvidiaSTTAdapter
|
||||
from voice_assistant.tts.voxcpm import VoxCPMTTSAdapter
|
||||
|
||||
|
||||
def test_create_stt_adapter_from_config():
|
||||
config = load_config()
|
||||
catalog = load_provider_catalog()
|
||||
|
||||
stt = create_stt_adapter(
|
||||
config=config,
|
||||
catalog=catalog,
|
||||
)
|
||||
|
||||
assert isinstance(stt, NvidiaSTTAdapter)
|
||||
assert stt.server == "127.0.0.1:9000"
|
||||
assert stt.language_code == "pt-BR"
|
||||
assert stt.automatic_punctuation is True
|
||||
|
||||
|
||||
def test_create_agent_adapter_from_config():
|
||||
config = load_config()
|
||||
catalog = load_provider_catalog()
|
||||
|
||||
agent = create_agent_adapter(
|
||||
config=config,
|
||||
catalog=catalog,
|
||||
)
|
||||
|
||||
assert isinstance(agent, MockAgentAdapter)
|
||||
|
||||
|
||||
def test_create_tts_adapter_from_config():
|
||||
config = load_config()
|
||||
catalog = load_provider_catalog()
|
||||
|
||||
tts = create_tts_adapter(
|
||||
config=config,
|
||||
catalog=catalog,
|
||||
)
|
||||
|
||||
assert isinstance(tts, VoxCPMTTSAdapter)
|
||||
assert tts.server == "http://127.0.0.1:8000"
|
||||
assert tts.model == "openbmb/VoxCPM2"
|
||||
assert tts.voice == "default"
|
||||
|
||||
75
tests/test_providers.py
Normal file
75
tests/test_providers.py
Normal file
@@ -0,0 +1,75 @@
|
||||
import pytest
|
||||
|
||||
|
||||
from voice_assistant.agent.mock import MockAgentAdapter
|
||||
from voice_assistant.config.loader import load_provider_catalog
|
||||
from voice_assistant.tts.voxcpm import VoxCPMTTSAdapter
|
||||
from voice_assistant.config.models import ProviderCatalog
|
||||
from voice_assistant.providers import load_provider_class
|
||||
|
||||
|
||||
def test_provider_must_implement_component_contract():
|
||||
catalog = ProviderCatalog.model_validate(
|
||||
{
|
||||
"providers": {
|
||||
"tts": {
|
||||
"invalid": {
|
||||
"name": "Invalid TTS Provider",
|
||||
"implementation": (
|
||||
"voice_assistant.agent.mock:"
|
||||
"MockAgentAdapter"
|
||||
),
|
||||
"capabilities": {},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
TypeError,
|
||||
match="must implement TTSAdapter",
|
||||
):
|
||||
load_provider_class(
|
||||
catalog=catalog,
|
||||
component="tts",
|
||||
provider="invalid",
|
||||
)
|
||||
|
||||
|
||||
def test_load_tts_provider_class():
|
||||
catalog = load_provider_catalog()
|
||||
|
||||
provider_class = load_provider_class(
|
||||
catalog=catalog,
|
||||
component="tts",
|
||||
provider="voxcpm",
|
||||
)
|
||||
|
||||
assert provider_class is VoxCPMTTSAdapter
|
||||
|
||||
|
||||
def test_load_agent_provider_class():
|
||||
catalog = load_provider_catalog()
|
||||
|
||||
provider_class = load_provider_class(
|
||||
catalog=catalog,
|
||||
component="agent",
|
||||
provider="mock",
|
||||
)
|
||||
|
||||
assert provider_class is MockAgentAdapter
|
||||
|
||||
|
||||
def test_unknown_provider_fails():
|
||||
catalog = load_provider_catalog()
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="is not registered",
|
||||
):
|
||||
load_provider_class(
|
||||
catalog=catalog,
|
||||
component="tts",
|
||||
provider="does-not-exist",
|
||||
)
|
||||
281
tests/test_session_turn_lifecycle.py
Normal file
281
tests/test_session_turn_lifecycle.py
Normal file
@@ -0,0 +1,281 @@
|
||||
"""
|
||||
Tests for SessionState and TurnManager lifecycle ownership.
|
||||
|
||||
Purpose:
|
||||
Define and verify the separation between session lifecycle state
|
||||
and semantic turn state.
|
||||
|
||||
Responsibilities:
|
||||
- Verify SessionState owns session and audio sequencing state.
|
||||
- Verify TurnManager owns semantic turn transitions.
|
||||
- Verify the expected lifecycle of a complete turn.
|
||||
- Verify interruption and invalid transitions.
|
||||
|
||||
Non-responsibilities:
|
||||
- Testing STT, Agent, or TTS providers.
|
||||
- Testing WebSocket transport.
|
||||
- Testing provider configuration.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from voice_assistant.audio_gateway.app.session import SessionState
|
||||
from voice_assistant.turn_manager import TurnManager, TurnState
|
||||
|
||||
|
||||
def test_session_creates_sequential_turn_ids():
|
||||
session = SessionState()
|
||||
|
||||
first_turn = session.start_turn()
|
||||
|
||||
assert first_turn == 1
|
||||
assert session.active_turn_id == 1
|
||||
assert session.expected_audio_sequence == 0
|
||||
|
||||
session.end_turn(first_turn)
|
||||
|
||||
second_turn = session.start_turn()
|
||||
|
||||
assert second_turn == 2
|
||||
assert session.active_turn_id == 2
|
||||
assert session.expected_audio_sequence == 0
|
||||
|
||||
|
||||
def test_session_owns_audio_sequence():
|
||||
session = SessionState()
|
||||
|
||||
turn_id = session.start_turn()
|
||||
|
||||
session.accept_audio_chunk(
|
||||
turn_id=turn_id,
|
||||
sequence_number=0,
|
||||
payload_size=320,
|
||||
)
|
||||
|
||||
assert session.expected_audio_sequence == 1
|
||||
|
||||
session.accept_audio_chunk(
|
||||
turn_id=turn_id,
|
||||
sequence_number=1,
|
||||
payload_size=320,
|
||||
)
|
||||
|
||||
assert session.expected_audio_sequence == 2
|
||||
|
||||
|
||||
def test_session_rejects_out_of_order_audio():
|
||||
session = SessionState()
|
||||
|
||||
turn_id = session.start_turn()
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Expected sequence_number=0",
|
||||
):
|
||||
session.accept_audio_chunk(
|
||||
turn_id=turn_id,
|
||||
sequence_number=1,
|
||||
payload_size=320,
|
||||
)
|
||||
|
||||
|
||||
def test_turn_manager_owns_semantic_turn_state():
|
||||
manager = TurnManager()
|
||||
|
||||
turn = manager.start(turn_id=1)
|
||||
|
||||
assert turn.turn_id == 1
|
||||
assert turn.state == TurnState.LISTENING
|
||||
|
||||
manager.on_partial(
|
||||
turn_id=1,
|
||||
text="Olá",
|
||||
)
|
||||
|
||||
assert turn.state == TurnState.LISTENING
|
||||
|
||||
manager.on_final(
|
||||
turn_id=1,
|
||||
text="Olá!",
|
||||
)
|
||||
|
||||
assert turn.state == TurnState.PROCESSING
|
||||
|
||||
manager.on_agent_response(turn_id=1)
|
||||
|
||||
assert turn.state == TurnState.SPEAKING
|
||||
|
||||
manager.on_tts_complete(turn_id=1)
|
||||
|
||||
assert turn.state == TurnState.COMPLETED
|
||||
|
||||
|
||||
def test_turn_manager_rejects_invalid_transition():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(turn_id=1)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="Cannot start speaking",
|
||||
):
|
||||
manager.on_agent_response(turn_id=1)
|
||||
|
||||
|
||||
def test_turn_manager_interrupts_active_turn():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(turn_id=1)
|
||||
|
||||
turn = manager.interrupt(turn_id=1)
|
||||
|
||||
assert turn.state == TurnState.INTERRUPTED
|
||||
assert manager.active_turn is turn
|
||||
|
||||
|
||||
def test_turn_manager_stop_requires_completed_state():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(turn_id=1)
|
||||
|
||||
manager.on_final(
|
||||
turn_id=1,
|
||||
text="Olá!",
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="Cannot stop turn while turn is processing",
|
||||
):
|
||||
manager.stop(turn_id=1)
|
||||
|
||||
|
||||
def test_turn_manager_stop_allows_completed_turn():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(turn_id=1)
|
||||
|
||||
manager.on_final(
|
||||
turn_id=1,
|
||||
text="Olá!",
|
||||
)
|
||||
|
||||
manager.on_agent_response(turn_id=1)
|
||||
|
||||
manager.on_tts_complete(turn_id=1)
|
||||
|
||||
turn = manager.stop(turn_id=1)
|
||||
|
||||
assert turn.state == TurnState.COMPLETED
|
||||
assert turn.final_text == "Olá!"
|
||||
|
||||
def test_session_and_turn_manager_complete_same_turn():
|
||||
session = SessionState()
|
||||
manager = TurnManager()
|
||||
|
||||
turn_id = session.start_turn()
|
||||
turn = manager.start(turn_id)
|
||||
|
||||
assert session.active_turn_id == turn_id
|
||||
assert turn.turn_id == turn_id
|
||||
assert turn.state == TurnState.LISTENING
|
||||
|
||||
manager.on_final(
|
||||
turn_id,
|
||||
"Olá!",
|
||||
)
|
||||
|
||||
manager.on_agent_response(turn_id)
|
||||
manager.on_tts_complete(turn_id)
|
||||
|
||||
completed_turn = manager.stop(turn_id)
|
||||
|
||||
assert completed_turn.state == TurnState.COMPLETED
|
||||
|
||||
manager.clear(turn_id)
|
||||
session.end_turn(turn_id)
|
||||
|
||||
assert manager.active_turn is None
|
||||
assert session.active_turn_id is None
|
||||
|
||||
def test_session_and_turn_manager_must_use_same_turn_id():
|
||||
session = SessionState()
|
||||
manager = TurnManager()
|
||||
|
||||
session_turn_id = session.start_turn()
|
||||
manager.start(1)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Unexpected turn_id=2",
|
||||
):
|
||||
session.accept_audio_chunk(
|
||||
turn_id=2,
|
||||
sequence_number=0,
|
||||
payload_size=320,
|
||||
)
|
||||
|
||||
assert session.active_turn_id == session_turn_id
|
||||
assert manager.active_turn is not None
|
||||
assert manager.active_turn.turn_id == 1
|
||||
|
||||
|
||||
def test_session_ends_turn_after_turn_manager_completion():
|
||||
session = SessionState()
|
||||
manager = TurnManager()
|
||||
|
||||
turn_id = session.start_turn()
|
||||
manager.start(turn_id)
|
||||
|
||||
manager.on_final(
|
||||
turn_id,
|
||||
"Olá!",
|
||||
)
|
||||
|
||||
manager.on_agent_response(turn_id)
|
||||
manager.on_tts_complete(turn_id)
|
||||
|
||||
manager.stop(turn_id)
|
||||
manager.clear(turn_id)
|
||||
session.end_turn(turn_id)
|
||||
|
||||
assert session.active_turn_id is None
|
||||
|
||||
|
||||
def test_session_and_turn_manager_interrupt_same_turn():
|
||||
session = SessionState()
|
||||
manager = TurnManager()
|
||||
|
||||
turn_id = session.start_turn()
|
||||
manager.start(turn_id)
|
||||
|
||||
session.interrupt(turn_id)
|
||||
interrupted_turn = manager.interrupt(turn_id)
|
||||
|
||||
assert session.active_turn_id is None
|
||||
assert manager.active_turn is interrupted_turn
|
||||
assert interrupted_turn.turn_id == turn_id
|
||||
assert interrupted_turn.state == TurnState.INTERRUPTED
|
||||
|
||||
|
||||
def test_session_can_start_next_turn_after_interrupt():
|
||||
session = SessionState()
|
||||
manager = TurnManager()
|
||||
|
||||
first_turn_id = session.start_turn()
|
||||
manager.start(first_turn_id)
|
||||
|
||||
session.interrupt(first_turn_id)
|
||||
manager.interrupt(first_turn_id)
|
||||
manager.clear(first_turn_id)
|
||||
|
||||
second_turn_id = session.start_turn()
|
||||
manager.start(second_turn_id)
|
||||
|
||||
assert second_turn_id == 2
|
||||
assert session.active_turn_id == 2
|
||||
assert manager.active_turn is not None
|
||||
assert manager.active_turn.turn_id == 2
|
||||
assert manager.active_turn.state == TurnState.LISTENING
|
||||
|
||||
|
||||
307
tests/test_stt_abort.py
Normal file
307
tests/test_stt_abort.py
Normal file
@@ -0,0 +1,307 @@
|
||||
"""
|
||||
Tests for STT session abort behavior.
|
||||
|
||||
Purpose:
|
||||
Verify that STT implementations support interruption
|
||||
without delivering stale transcription results.
|
||||
|
||||
Responsibilities:
|
||||
- Validate immediate abort.
|
||||
- Verify pending transcript events are discarded.
|
||||
- Verify repeated abort calls are safe.
|
||||
- Verify audio is rejected after abort.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from voice_assistant.stt.nvidia import NvidiaSTTAdapter
|
||||
from voice_assistant.stt.models import TranscriptEvent
|
||||
|
||||
from voice_assistant.stt.mock import MockSTTAdapter
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mock_stt_abort_discards_pending_events():
|
||||
stt = MockSTTAdapter()
|
||||
|
||||
await stt.start_session()
|
||||
|
||||
await stt.emit_partial("Olá")
|
||||
await stt.emit_final("Olá, tudo bem?")
|
||||
|
||||
await stt.abort_session()
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in stt.events()
|
||||
]
|
||||
|
||||
assert events == []
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mock_stt_abort_rejects_audio():
|
||||
stt = MockSTTAdapter()
|
||||
|
||||
await stt.start_session()
|
||||
await stt.abort_session()
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="STT session has not been started",
|
||||
):
|
||||
await stt.send_audio(b"\x00\x01")
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mock_stt_abort_is_idempotent():
|
||||
stt = MockSTTAdapter()
|
||||
|
||||
await stt.start_session()
|
||||
|
||||
await stt.abort_session()
|
||||
await stt.abort_session()
|
||||
|
||||
assert stt._started is False
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mock_stt_abort_unblocks_event_consumer():
|
||||
stt = MockSTTAdapter()
|
||||
|
||||
await stt.start_session()
|
||||
|
||||
async def consume():
|
||||
return [
|
||||
event
|
||||
async for event in stt.events()
|
||||
]
|
||||
|
||||
import asyncio
|
||||
|
||||
consumer = asyncio.create_task(consume())
|
||||
|
||||
# Allow the consumer to start waiting for events.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
await stt.abort_session()
|
||||
|
||||
events = await asyncio.wait_for(
|
||||
consumer,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert events == []
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_nvidia_abort_without_started_session():
|
||||
stt = NvidiaSTTAdapter()
|
||||
|
||||
await stt.abort_session()
|
||||
await stt.abort_session()
|
||||
|
||||
assert stt._started is False
|
||||
assert stt._websocket is None
|
||||
assert stt._http is None
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_nvidia_abort_discards_pending_events():
|
||||
stt = NvidiaSTTAdapter()
|
||||
|
||||
stt._started = True
|
||||
|
||||
await stt._events.put(
|
||||
TranscriptEvent(
|
||||
type="partial",
|
||||
text="Transcrição antiga",
|
||||
)
|
||||
)
|
||||
|
||||
await stt.abort_session()
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in stt.events()
|
||||
]
|
||||
|
||||
assert events == []
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_nvidia_abort_unblocks_consumer():
|
||||
stt = NvidiaSTTAdapter()
|
||||
stt._started = True
|
||||
|
||||
async def consume():
|
||||
return [
|
||||
event
|
||||
async for event in stt.events()
|
||||
]
|
||||
|
||||
consumer = asyncio.create_task(consume())
|
||||
|
||||
await asyncio.sleep(0)
|
||||
|
||||
await stt.abort_session()
|
||||
|
||||
events = await asyncio.wait_for(
|
||||
consumer,
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
assert events == []
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_nvidia_abort_closes_resources_without_done():
|
||||
stt = NvidiaSTTAdapter()
|
||||
|
||||
websocket = AsyncMock()
|
||||
http_client = AsyncMock()
|
||||
|
||||
stt._started = True
|
||||
stt._websocket = websocket
|
||||
stt._http = http_client
|
||||
|
||||
await stt.abort_session()
|
||||
|
||||
websocket.close.assert_awaited_once()
|
||||
websocket.send.assert_not_awaited()
|
||||
http_client.aclose.assert_awaited_once()
|
||||
|
||||
assert stt._websocket is None
|
||||
assert stt._http is None
|
||||
assert stt._started is False
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_nvidia_abort_cancels_receiver_task():
|
||||
stt = NvidiaSTTAdapter()
|
||||
stt._started = True
|
||||
|
||||
started = asyncio.Event()
|
||||
finished = asyncio.Event()
|
||||
|
||||
async def receiver():
|
||||
started.set()
|
||||
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
finished.set()
|
||||
|
||||
task = asyncio.create_task(receiver())
|
||||
stt._receiver_task = task
|
||||
|
||||
await started.wait()
|
||||
|
||||
await stt.abort_session()
|
||||
|
||||
assert task.done()
|
||||
assert task.cancelled()
|
||||
assert finished.is_set()
|
||||
assert stt._receiver_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_nvidia_concurrent_abort():
|
||||
"""
|
||||
Verify that concurrent abort requests produce a single
|
||||
end-of-stream marker and leave the adapter stopped.
|
||||
"""
|
||||
stt = NvidiaSTTAdapter()
|
||||
stt._started = True
|
||||
|
||||
await asyncio.gather(
|
||||
stt.abort_session(),
|
||||
stt.abort_session(),
|
||||
)
|
||||
|
||||
assert stt._started is False
|
||||
assert stt._receiver_task is None
|
||||
assert stt._websocket is None
|
||||
assert stt._http is None
|
||||
|
||||
assert stt._events.qsize() == 1
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_nvidia_abort_does_not_deliver_late_events():
|
||||
"""Queued transcripts must not survive an explicit abort."""
|
||||
stt = NvidiaSTTAdapter()
|
||||
stt._started = True
|
||||
|
||||
await stt._events.put(
|
||||
TranscriptEvent(
|
||||
type="partial",
|
||||
text="Resultado antigo",
|
||||
)
|
||||
)
|
||||
|
||||
await stt.abort_session()
|
||||
|
||||
events = [
|
||||
event
|
||||
async for event in stt.events()
|
||||
]
|
||||
|
||||
assert events == []
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_nvidia_concurrent_abort_closes_resources_once():
|
||||
"""
|
||||
Verify that concurrent abort requests do not close
|
||||
provider resources more than once.
|
||||
"""
|
||||
stt = NvidiaSTTAdapter()
|
||||
|
||||
websocket = AsyncMock()
|
||||
http_client = AsyncMock()
|
||||
|
||||
stt._started = True
|
||||
stt._websocket = websocket
|
||||
stt._http = http_client
|
||||
|
||||
# Force both abort operations to overlap during cleanup.
|
||||
close_started = asyncio.Event()
|
||||
allow_close = asyncio.Event()
|
||||
|
||||
async def slow_close():
|
||||
close_started.set()
|
||||
await allow_close.wait()
|
||||
|
||||
websocket.close.side_effect = slow_close
|
||||
|
||||
first_abort = asyncio.create_task(
|
||||
stt.abort_session()
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
close_started.wait(),
|
||||
timeout=1.0,
|
||||
)
|
||||
|
||||
second_abort = asyncio.create_task(
|
||||
stt.abort_session()
|
||||
)
|
||||
|
||||
# Give the second abort a chance to enter cleanup.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
finally:
|
||||
allow_close.set()
|
||||
|
||||
await asyncio.gather(
|
||||
first_abort,
|
||||
second_abort,
|
||||
)
|
||||
|
||||
websocket.close.assert_awaited_once()
|
||||
http_client.aclose.assert_awaited_once()
|
||||
|
||||
assert stt._events.qsize() == 1
|
||||
39
tests/test_tts.py
Normal file
39
tests/test_tts.py
Normal file
@@ -0,0 +1,39 @@
|
||||
import pytest
|
||||
|
||||
from voice_assistant.stt.mock import MockSTTAdapter
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mock_stt_lifecycle():
|
||||
stt = MockSTTAdapter()
|
||||
|
||||
await stt.start_session()
|
||||
|
||||
await stt.send_audio(b"\x00\x00" * 160)
|
||||
|
||||
await stt.emit_partial("eu gostaria de saber")
|
||||
await stt.emit_final(
|
||||
"eu gostaria de saber o horário de funcionamento"
|
||||
)
|
||||
|
||||
await stt.stop_session()
|
||||
|
||||
events = [event async for event in stt.events()]
|
||||
|
||||
assert len(events) == 2
|
||||
|
||||
assert events[0].type == "partial"
|
||||
assert events[0].text == "eu gostaria de saber"
|
||||
|
||||
assert events[1].type == "final"
|
||||
assert events[1].text == (
|
||||
"eu gostaria de saber o horário de funcionamento"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mock_stt_rejects_audio_before_start():
|
||||
stt = MockSTTAdapter()
|
||||
|
||||
with pytest.raises(RuntimeError, match="has not been started"):
|
||||
await stt.send_audio(b"\x00\x00")
|
||||
32
tests/test_tts_adapter.py
Normal file
32
tests/test_tts_adapter.py
Normal file
@@ -0,0 +1,32 @@
|
||||
import pytest
|
||||
|
||||
from voice_assistant.tts import MockTTSAdapter
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mock_tts_synthesizes_audio_chunks():
|
||||
tts= MockTTSAdapter()
|
||||
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in tts.synthesize("Olá, tudo bem?")
|
||||
]
|
||||
|
||||
assert chunks == [
|
||||
b"\x01\x02\x03\x04",
|
||||
b"\x05\x06\x07\x08",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_mock_tts_records_text():
|
||||
tts = MockTTSAdapter()
|
||||
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in tts.synthesize("Olá")
|
||||
]
|
||||
|
||||
assert chunks
|
||||
assert tts.calls == ["Olá"], "A mensagem de retorno deve ser Olá"
|
||||
|
||||
194
tests/test_turn_manager.py
Normal file
194
tests/test_turn_manager.py
Normal file
@@ -0,0 +1,194 @@
|
||||
import pytest
|
||||
|
||||
from voice_assistant.turn_manager import (
|
||||
TurnManager,
|
||||
TurnState,
|
||||
)
|
||||
|
||||
|
||||
def test_start_turn():
|
||||
manager = TurnManager()
|
||||
|
||||
turn = manager.start(1)
|
||||
|
||||
assert turn.turn_id == 1
|
||||
assert turn.state == TurnState.LISTENING
|
||||
assert turn.partial_text == ""
|
||||
assert turn.final_text is None
|
||||
|
||||
|
||||
def test_partial_replaces_previous_partial():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
|
||||
manager.on_partial(1, "Ô Alex")
|
||||
assert manager.active_turn is not None
|
||||
assert manager.active_turn.partial_text == "Ô Alex"
|
||||
|
||||
manager.on_partial(1, "Ô Alex tudo bem")
|
||||
assert manager.active_turn.partial_text == "Ô Alex tudo bem"
|
||||
|
||||
|
||||
def test_final_moves_turn_to_processing():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
|
||||
manager.on_partial(1, "Ô Alex tudo bem")
|
||||
manager.on_final(1, "Ô Alex tudo bem?")
|
||||
|
||||
assert manager.active_turn is not None
|
||||
assert manager.active_turn.final_text == "Ô Alex tudo bem?"
|
||||
assert manager.active_turn.partial_text == "Ô Alex tudo bem?"
|
||||
assert manager.active_turn.state == TurnState.PROCESSING
|
||||
|
||||
|
||||
def test_cannot_start_second_turn():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="A turn is already active",
|
||||
):
|
||||
manager.start(2)
|
||||
|
||||
|
||||
def test_cannot_stop_before_completed():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="final transcript",
|
||||
):
|
||||
manager.stop(1)
|
||||
|
||||
|
||||
def test_interrupt():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
manager.interrupt(1)
|
||||
|
||||
assert manager.active_turn is not None
|
||||
assert (
|
||||
manager.active_turn.state
|
||||
== TurnState.INTERRUPTED
|
||||
)
|
||||
|
||||
|
||||
def test_clear_turn():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
manager.clear(1)
|
||||
|
||||
assert manager.active_turn is None
|
||||
|
||||
|
||||
def test_agent_response_moves_turn_to_speaking():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
manager.on_final(
|
||||
1,
|
||||
"Qual o horário?",
|
||||
)
|
||||
|
||||
manager.on_agent_response(1)
|
||||
|
||||
assert manager.active_turn is not None
|
||||
assert (
|
||||
manager.active_turn.state
|
||||
== TurnState.SPEAKING
|
||||
)
|
||||
|
||||
|
||||
def test_tts_completion_moves_turn_to_completed():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
|
||||
manager.on_final(
|
||||
1,
|
||||
"Qual o horário?",
|
||||
)
|
||||
|
||||
manager.on_agent_response(1)
|
||||
manager.on_tts_complete(1)
|
||||
|
||||
assert manager.active_turn is not None
|
||||
assert (
|
||||
manager.active_turn.state
|
||||
== TurnState.COMPLETED
|
||||
)
|
||||
|
||||
|
||||
def test_agent_response_requires_processing_state():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="turn is listening",
|
||||
):
|
||||
manager.on_agent_response(1)
|
||||
|
||||
|
||||
def test_tts_completion_requires_speaking_state():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="turn is listening",
|
||||
):
|
||||
manager.on_tts_complete(1)
|
||||
|
||||
|
||||
def test_cannot_stop_before_completed():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="Cannot stop turn while turn is listening",
|
||||
):
|
||||
manager.stop(1)
|
||||
|
||||
|
||||
def test_clear_removes_completed_turn():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
|
||||
manager.on_final(
|
||||
1,
|
||||
"Olá!",
|
||||
)
|
||||
|
||||
manager.on_agent_response(1)
|
||||
manager.on_tts_complete(1)
|
||||
|
||||
manager.clear(1)
|
||||
|
||||
assert manager.active_turn is None
|
||||
|
||||
|
||||
def test_clear_rejects_unexpected_turn_id():
|
||||
manager = TurnManager()
|
||||
|
||||
manager.start(1)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Unexpected turn_id=2",
|
||||
):
|
||||
manager.clear(2)
|
||||
820
tests/test_voice_pipeline.py
Normal file
820
tests/test_voice_pipeline.py
Normal file
@@ -0,0 +1,820 @@
|
||||
"""
|
||||
Tests for the Voice Pipeline lifecycle orchestration.
|
||||
|
||||
Purpose:
|
||||
Verify that the VoicePipeline coordinates SessionState and
|
||||
TurnManager without coupling either component to transport details.
|
||||
|
||||
Responsibilities:
|
||||
- Validate turn creation.
|
||||
- Validate semantic turn completion.
|
||||
- Validate coordinated interruption.
|
||||
- Verify SessionState and TurnManager remain independently modeled.
|
||||
|
||||
Non-responsibilities:
|
||||
- Testing WebSocket transport.
|
||||
- Testing STT, Agent, or TTS providers.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
|
||||
from voice_assistant.audio_gateway.app.session import SessionState
|
||||
from voice_assistant.turn_manager import TurnManager, TurnState
|
||||
from voice_assistant.voice_pipeline import VoicePipeline
|
||||
from voice_assistant.agent.mock import MockAgentAdapter
|
||||
from voice_assistant.tts.mock import MockTTSAdapter
|
||||
|
||||
|
||||
def create_pipeline() -> tuple[
|
||||
VoicePipeline,
|
||||
SessionState,
|
||||
TurnManager,
|
||||
MockAgentAdapter,
|
||||
MockTTSAdapter,
|
||||
]:
|
||||
session = SessionState()
|
||||
turn_manager = TurnManager()
|
||||
agent = MockAgentAdapter()
|
||||
tts = MockTTSAdapter()
|
||||
|
||||
pipeline = VoicePipeline(
|
||||
session=session,
|
||||
turn_manager=turn_manager,
|
||||
agent=agent,
|
||||
tts=tts,
|
||||
)
|
||||
|
||||
return (
|
||||
pipeline,
|
||||
session,
|
||||
turn_manager,
|
||||
agent,
|
||||
tts,
|
||||
)
|
||||
|
||||
|
||||
def test_pipeline_starts_turn():
|
||||
pipeline, session, turn_manager, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
assert turn.turn_id == 1
|
||||
assert turn.state == TurnState.LISTENING
|
||||
|
||||
assert session.active_turn_id == 1
|
||||
assert turn_manager.active_turn is turn
|
||||
|
||||
|
||||
def test_pipeline_completes_turn():
|
||||
pipeline, session, turn_manager, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
turn_manager.on_final(
|
||||
turn.turn_id,
|
||||
"Olá!",
|
||||
)
|
||||
turn_manager.on_agent_response(turn.turn_id)
|
||||
turn_manager.on_tts_complete(turn.turn_id)
|
||||
|
||||
completed_turn = pipeline.complete_turn(
|
||||
turn.turn_id,
|
||||
)
|
||||
|
||||
assert completed_turn.state == TurnState.COMPLETED
|
||||
assert session.active_turn_id is None
|
||||
assert turn_manager.active_turn is None
|
||||
|
||||
|
||||
def test_pipeline_rejects_incomplete_turn():
|
||||
pipeline, session, turn_manager, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="Cannot stop turn while turn is listening",
|
||||
):
|
||||
pipeline.complete_turn(turn.turn_id)
|
||||
|
||||
# Failed completion must not destroy the active turn.
|
||||
assert session.active_turn_id == turn.turn_id
|
||||
assert turn_manager.active_turn is turn
|
||||
|
||||
|
||||
def test_pipeline_interrupts_turn():
|
||||
pipeline, session, turn_manager, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
interrupted_turn = pipeline.interrupt_turn(
|
||||
turn.turn_id,
|
||||
)
|
||||
|
||||
assert interrupted_turn.state == TurnState.INTERRUPTED
|
||||
assert session.active_turn_id is None
|
||||
assert turn_manager.active_turn is None
|
||||
|
||||
|
||||
def test_pipeline_can_start_turn_after_interrupt():
|
||||
pipeline, session, turn_manager, _, _ = create_pipeline()
|
||||
|
||||
first_turn = pipeline.start_turn()
|
||||
|
||||
pipeline.interrupt_turn(
|
||||
first_turn.turn_id,
|
||||
)
|
||||
|
||||
second_turn = pipeline.start_turn()
|
||||
|
||||
assert second_turn.turn_id == 2
|
||||
assert second_turn.state == TurnState.LISTENING
|
||||
assert session.active_turn_id == 2
|
||||
assert turn_manager.active_turn is second_turn
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_processes_transcript_through_agent():
|
||||
(
|
||||
pipeline,
|
||||
session,
|
||||
turn_manager,
|
||||
agent,
|
||||
_,
|
||||
) = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
turn_manager.on_final(
|
||||
turn.turn_id,
|
||||
"Olá!",
|
||||
)
|
||||
|
||||
result = await pipeline.process_transcript(
|
||||
session_id=session.session_id,
|
||||
turn_id=turn.turn_id,
|
||||
text="Olá!",
|
||||
)
|
||||
|
||||
assert result.response_text == (
|
||||
"Resposta simulada para: Olá!"
|
||||
)
|
||||
|
||||
assert agent.calls == [
|
||||
(
|
||||
session.session_id,
|
||||
turn.turn_id,
|
||||
"Olá!",
|
||||
)
|
||||
]
|
||||
|
||||
assert turn_manager.active_turn is not None
|
||||
assert (
|
||||
turn_manager.active_turn.state
|
||||
== TurnState.SPEAKING
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_streams_tts_audio():
|
||||
(
|
||||
pipeline,
|
||||
session,
|
||||
turn_manager,
|
||||
_,
|
||||
tts,
|
||||
) = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
turn_manager.on_final(
|
||||
turn.turn_id,
|
||||
"Olá!",
|
||||
)
|
||||
|
||||
result = await pipeline.process_transcript(
|
||||
session_id=session.session_id,
|
||||
turn_id=turn.turn_id,
|
||||
text="Olá!",
|
||||
)
|
||||
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in pipeline.synthesize_response(
|
||||
result.response_text,
|
||||
)
|
||||
]
|
||||
|
||||
assert chunks == [
|
||||
b"\x01\x02\x03\x04",
|
||||
b"\x05\x06\x07\x08",
|
||||
]
|
||||
|
||||
assert tts.calls == [
|
||||
"Resposta simulada para: Olá!",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_processes_complete_voice_turn():
|
||||
(
|
||||
pipeline,
|
||||
session,
|
||||
turn_manager,
|
||||
agent,
|
||||
tts,
|
||||
) = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
turn_manager.on_final(
|
||||
turn.turn_id,
|
||||
"Olá!",
|
||||
)
|
||||
|
||||
result = await pipeline.process_transcript(
|
||||
session_id=session.session_id,
|
||||
turn_id=turn.turn_id,
|
||||
text="Olá!",
|
||||
)
|
||||
|
||||
audio_chunks = [
|
||||
chunk
|
||||
async for chunk in pipeline.synthesize_response(
|
||||
result.response_text,
|
||||
)
|
||||
]
|
||||
|
||||
turn_manager.on_tts_complete(
|
||||
turn.turn_id,
|
||||
)
|
||||
|
||||
assert result.response_text == (
|
||||
"Resposta simulada para: Olá!"
|
||||
)
|
||||
|
||||
assert audio_chunks == [
|
||||
b"\x01\x02\x03\x04",
|
||||
b"\x05\x06\x07\x08",
|
||||
]
|
||||
|
||||
assert turn_manager.active_turn is not None
|
||||
assert (
|
||||
turn_manager.active_turn.state
|
||||
== TurnState.COMPLETED
|
||||
)
|
||||
|
||||
assert agent.calls == [
|
||||
(
|
||||
session.session_id,
|
||||
turn.turn_id,
|
||||
"Olá!",
|
||||
)
|
||||
]
|
||||
|
||||
assert tts.calls == [
|
||||
"Resposta simulada para: Olá!",
|
||||
]
|
||||
|
||||
|
||||
def test_pipeline_handles_partial_transcript():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
pipeline.on_partial(
|
||||
turn.turn_id,
|
||||
"Olá",
|
||||
)
|
||||
|
||||
assert turn.partial_text == "Olá"
|
||||
assert turn.state == TurnState.LISTENING
|
||||
|
||||
|
||||
def test_pipeline_handles_final_transcript():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
pipeline.on_final(
|
||||
turn.turn_id,
|
||||
"Olá!",
|
||||
)
|
||||
|
||||
assert turn.final_text == "Olá!"
|
||||
assert turn.partial_text == "Olá!"
|
||||
assert turn.state == TurnState.PROCESSING
|
||||
|
||||
|
||||
def test_pipeline_handles_tts_completion():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
pipeline.on_final(
|
||||
turn.turn_id,
|
||||
"Olá!",
|
||||
)
|
||||
|
||||
# The Agent transition is part of the normal semantic lifecycle.
|
||||
pipeline.on_agent_response(
|
||||
turn.turn_id,
|
||||
)
|
||||
|
||||
pipeline.on_tts_complete(
|
||||
turn.turn_id,
|
||||
)
|
||||
|
||||
assert turn.state == TurnState.COMPLETED
|
||||
|
||||
|
||||
def test_pipeline_closes_session_without_active_turn():
|
||||
pipeline, session, turn_manager, _, _ = create_pipeline()
|
||||
|
||||
pipeline.close_session()
|
||||
|
||||
assert session.closed is True
|
||||
assert session.active_turn_id is None
|
||||
assert turn_manager.active_turn is None
|
||||
|
||||
|
||||
def test_pipeline_closes_session_with_active_turn():
|
||||
pipeline, session, turn_manager, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
pipeline.close_session()
|
||||
|
||||
assert session.closed is True
|
||||
assert session.active_turn_id is None
|
||||
assert turn_manager.active_turn is None
|
||||
assert turn.state == TurnState.INTERRUPTED
|
||||
|
||||
|
||||
def test_pipeline_cannot_start_turn_after_session_close():
|
||||
pipeline, session, turn_manager, _, _ = create_pipeline()
|
||||
|
||||
pipeline.close_session()
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="Session is closed",
|
||||
):
|
||||
pipeline.start_turn()
|
||||
|
||||
|
||||
|
||||
def test_pipeline_interrupt_if_active():
|
||||
pipeline, session, turn_manager, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
interrupted = pipeline.interrupt_if_active(
|
||||
turn.turn_id,
|
||||
)
|
||||
|
||||
assert interrupted is True
|
||||
assert session.active_turn_id is None
|
||||
assert turn_manager.active_turn is None
|
||||
assert turn.state == TurnState.INTERRUPTED
|
||||
|
||||
|
||||
def test_pipeline_interrupt_if_active_unknown_turn():
|
||||
pipeline, session, turn_manager, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
interrupted = pipeline.interrupt_if_active(
|
||||
turn.turn_id + 1,
|
||||
)
|
||||
|
||||
assert interrupted is False
|
||||
assert session.active_turn_id == turn.turn_id
|
||||
assert turn_manager.active_turn is turn
|
||||
assert turn.state == TurnState.LISTENING
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_registers_turn_task():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
pipeline.start_turn()
|
||||
|
||||
async def worker():
|
||||
await asyncio.sleep(0)
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
assert pipeline.turn_task is task
|
||||
|
||||
await task
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_rejects_duplicate_running_task():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
pipeline.start_turn()
|
||||
|
||||
started = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
|
||||
async def worker():
|
||||
started.set()
|
||||
await release.wait()
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
|
||||
try:
|
||||
await started.wait()
|
||||
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="A turn execution task is already active",
|
||||
):
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
finally:
|
||||
release.set()
|
||||
await task
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_rejects_task_without_active_turn():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
async def worker():
|
||||
await asyncio.sleep(0)
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
|
||||
try:
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="Cannot register task without an active turn",
|
||||
):
|
||||
pipeline.register_turn_task(task)
|
||||
finally:
|
||||
await task
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_cancels_running_turn_task():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
pipeline.start_turn()
|
||||
|
||||
started = asyncio.Event()
|
||||
cancelled = asyncio.Event()
|
||||
|
||||
async def worker():
|
||||
started.set()
|
||||
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
cancelled.set()
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
|
||||
await started.wait()
|
||||
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
await pipeline.cancel_turn_task()
|
||||
|
||||
assert cancelled.is_set()
|
||||
assert task.done()
|
||||
assert task.cancelled()
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_cancel_without_registered_task():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
await pipeline.cancel_turn_task()
|
||||
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_cancel_completed_task():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
pipeline.start_turn()
|
||||
|
||||
async def worker():
|
||||
return None
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
await task
|
||||
await pipeline.cancel_turn_task()
|
||||
|
||||
assert task.done()
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_rejects_self_cancellation():
|
||||
"""
|
||||
Verify that a turn task cannot cancel itself.
|
||||
|
||||
The task reference must remain available so another
|
||||
task can subsequently perform cleanup.
|
||||
"""
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
pipeline.start_turn()
|
||||
|
||||
async def worker():
|
||||
await pipeline.cancel_turn_task()
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
# Self-cancellation must be rejected.
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="A turn task cannot cancel itself",
|
||||
):
|
||||
await task
|
||||
|
||||
# The failed task remains registered until explicit cleanup.
|
||||
assert pipeline.turn_task is task
|
||||
|
||||
# Cleanup is performed from the test task, not the worker.
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="A turn task cannot cancel itself",
|
||||
):
|
||||
await pipeline.cancel_turn_task()
|
||||
|
||||
# The reference must be released after the completed task
|
||||
# has been awaited, even when its exception is propagated.
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_propagates_turn_task_failure():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
pipeline.start_turn()
|
||||
|
||||
async def worker():
|
||||
raise ValueError(
|
||||
"Provider execution failed."
|
||||
)
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
# Allow the worker to execute and finish with its error.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert task.done()
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Provider execution failed",
|
||||
):
|
||||
await pipeline.cancel_turn_task()
|
||||
|
||||
assert task.done()
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_propagates_waiter_cancellation():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
pipeline.start_turn()
|
||||
|
||||
worker_started = asyncio.Event()
|
||||
allow_cleanup = asyncio.Event()
|
||||
|
||||
async def worker():
|
||||
worker_started.set()
|
||||
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
await allow_cleanup.wait()
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
|
||||
await worker_started.wait()
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
waiter = asyncio.create_task(
|
||||
pipeline.cancel_turn_task()
|
||||
)
|
||||
|
||||
try:
|
||||
# Allow cancellation to reach the worker.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
waiter.cancel()
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await waiter
|
||||
|
||||
# The worker is still completing its cancellation cleanup.
|
||||
assert pipeline.turn_task is task
|
||||
|
||||
finally:
|
||||
allow_cleanup.set()
|
||||
|
||||
await pipeline.cancel_turn_task()
|
||||
|
||||
assert task.done()
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
def test_pipeline_identifies_active_turn():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
assert pipeline.is_active_turn(turn.turn_id) is True
|
||||
|
||||
|
||||
def test_pipeline_rejects_inactive_turn_id():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
turn = pipeline.start_turn()
|
||||
|
||||
assert pipeline.is_active_turn(turn.turn_id + 1) is False
|
||||
|
||||
pipeline.interrupt_turn(turn.turn_id)
|
||||
|
||||
assert pipeline.is_active_turn(turn.turn_id) is False
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_waits_for_completed_turn_task():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
pipeline.start_turn()
|
||||
|
||||
completed = asyncio.Event()
|
||||
|
||||
async def worker():
|
||||
completed.set()
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
await pipeline.wait_turn_task()
|
||||
|
||||
assert completed.is_set()
|
||||
assert task.done()
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_wait_propagates_task_failure():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
pipeline.start_turn()
|
||||
|
||||
async def worker():
|
||||
raise ValueError("Agent processing failed.")
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Agent processing failed",
|
||||
):
|
||||
await pipeline.wait_turn_task()
|
||||
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_wait_without_registered_task():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
await pipeline.wait_turn_task()
|
||||
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_cancel_rejects_invalid_timeout():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Turn cancellation timeout must be positive",
|
||||
):
|
||||
await pipeline.cancel_turn_task(timeout=0)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_cancel_completes_within_timeout():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
pipeline.start_turn()
|
||||
|
||||
started = asyncio.Event()
|
||||
|
||||
async def worker():
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
|
||||
await started.wait()
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
await pipeline.cancel_turn_task(timeout=1.0)
|
||||
|
||||
assert task.done()
|
||||
assert task.cancelled()
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_cancel_timeout_preserves_task_ownership():
|
||||
pipeline, _, _, _, _ = create_pipeline()
|
||||
pipeline.start_turn()
|
||||
|
||||
started = asyncio.Event()
|
||||
allow_cleanup = asyncio.Event()
|
||||
cancellation_received = asyncio.Event()
|
||||
|
||||
async def worker():
|
||||
started.set()
|
||||
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
|
||||
finally:
|
||||
cancellation_received.set()
|
||||
await allow_cleanup.wait()
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
|
||||
await started.wait()
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
try:
|
||||
with pytest.raises(
|
||||
TimeoutError,
|
||||
match="Turn task cancellation timed out",
|
||||
):
|
||||
await pipeline.cancel_turn_task(timeout=0.01)
|
||||
|
||||
assert cancellation_received.is_set()
|
||||
assert not task.done()
|
||||
assert pipeline.turn_task is task
|
||||
|
||||
finally:
|
||||
allow_cleanup.set()
|
||||
await pipeline.cancel_turn_task(timeout=1.0)
|
||||
|
||||
assert task.done()
|
||||
assert pipeline.turn_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_pipeline_rejects_new_turn_while_previous_task_runs():
|
||||
pipeline, session, _, _, _ = create_pipeline()
|
||||
|
||||
first_turn = pipeline.start_turn()
|
||||
|
||||
started = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
|
||||
async def worker():
|
||||
started.set()
|
||||
await release.wait()
|
||||
|
||||
task = asyncio.create_task(worker())
|
||||
|
||||
await started.wait()
|
||||
pipeline.register_turn_task(task)
|
||||
|
||||
# Simulate semantic interruption without task cancellation.
|
||||
pipeline.interrupt_turn(first_turn.turn_id)
|
||||
|
||||
try:
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="previous turn task is still running",
|
||||
):
|
||||
pipeline.start_turn()
|
||||
|
||||
assert session.active_turn_id is None
|
||||
|
||||
finally:
|
||||
release.set()
|
||||
await pipeline.wait_turn_task()
|
||||
|
||||
second_turn = pipeline.start_turn()
|
||||
|
||||
assert second_turn.turn_id == 2
|
||||
assert session.active_turn_id == 2
|
||||
|
||||
147
tests/test_voxcpm_tts.py
Normal file
147
tests/test_voxcpm_tts.py
Normal file
@@ -0,0 +1,147 @@
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from voice_assistant.tts.voxcpm import VoxCPMTTSAdapter
|
||||
|
||||
|
||||
class MockHTTPStream:
|
||||
def __init__(
|
||||
self,
|
||||
chunks: list[bytes],
|
||||
status_code: int = 200,
|
||||
) -> None:
|
||||
self.chunks = chunks
|
||||
self.status_code = status_code
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def raise_for_status(self):
|
||||
if self.status_code >= 400:
|
||||
raise httpx.HTTPStatusError(
|
||||
"mock error",
|
||||
request=httpx.Request("POST", "http://test"),
|
||||
response=httpx.Response(
|
||||
status_code=self.status_code,
|
||||
),
|
||||
)
|
||||
|
||||
async def aiter_bytes(self):
|
||||
for chunk in self.chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
class MockHTTPClient:
|
||||
last_request = None
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def stream(self, method, url, json):
|
||||
self.last_request = {
|
||||
"method": method,
|
||||
"url": url,
|
||||
"json": json,
|
||||
}
|
||||
|
||||
MockHTTPClient.last_request = self.last_request
|
||||
|
||||
return MockHTTPStream(
|
||||
[
|
||||
b"\x01\x02\x03\x04",
|
||||
b"\x05\x06\x07\x08",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_voxcpm_tts_streams_audio(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
httpx,
|
||||
"AsyncClient",
|
||||
MockHTTPClient,
|
||||
)
|
||||
|
||||
tts = VoxCPMTTSAdapter(
|
||||
server="http://127.0.0.1:8000",
|
||||
model="openbmb/VoxCPM2",
|
||||
voice="default",
|
||||
)
|
||||
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in tts.synthesize(
|
||||
"Olá Alex, tudo bem?"
|
||||
)
|
||||
]
|
||||
|
||||
assert chunks == [
|
||||
b"\x01\x02\x03\x04",
|
||||
b"\x05\x06\x07\x08",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_voxcpm_tts_sends_expected_payload(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
httpx,
|
||||
"AsyncClient",
|
||||
MockHTTPClient,
|
||||
)
|
||||
|
||||
tts = VoxCPMTTSAdapter(
|
||||
server="http://127.0.0.1:8000",
|
||||
model="openbmb/VoxCPM2",
|
||||
voice="default",
|
||||
)
|
||||
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in tts.synthesize("Olá")
|
||||
]
|
||||
|
||||
assert chunks
|
||||
|
||||
request = MockHTTPClient.last_request
|
||||
|
||||
assert request is not None
|
||||
|
||||
assert request["method"] == "POST"
|
||||
assert request["url"] == (
|
||||
"http://127.0.0.1:8000/v1/audio/speech"
|
||||
)
|
||||
assert request["json"] == {
|
||||
"input": "Olá",
|
||||
"model": "openbmb/VoxCPM2",
|
||||
"voice": "default",
|
||||
"response_format": "pcm",
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_voxcpm_tts_rejects_empty_text():
|
||||
tts = VoxCPMTTSAdapter(
|
||||
server="http://127.0.0.1:8000",
|
||||
model="openbmb/VoxCPM2",
|
||||
voice="default",
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Text cannot be empty",
|
||||
):
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in tts.synthesize("")
|
||||
]
|
||||
29
tests/test_voxcpm_tts_integration.py
Normal file
29
tests/test_voxcpm_tts_integration.py
Normal file
@@ -0,0 +1,29 @@
|
||||
import pytest
|
||||
|
||||
from voice_assistant.tts.voxcpm import VoxCPMTTSAdapter
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_voxcpm_tts_real():
|
||||
tts = VoxCPMTTSAdapter(
|
||||
server="http://127.0.0.1:8000",
|
||||
model="openbmb/VoxCPM2",
|
||||
voice="default",
|
||||
)
|
||||
|
||||
chunks = [
|
||||
chunk
|
||||
async for chunk in tts.synthesize(
|
||||
"Olá Alex, este é um teste do VoxCPM."
|
||||
)
|
||||
]
|
||||
|
||||
assert chunks
|
||||
assert all(isinstance(chunk, bytes) for chunk in chunks)
|
||||
|
||||
total_bytes = sum(len(chunk) for chunk in chunks)
|
||||
|
||||
assert total_bytes > 0
|
||||
|
||||
print(f"Chunks recebidos : {len(chunks)}")
|
||||
print(f"Bytes recebidos : {total_bytes}")
|
||||
246
tools/stt_gateway_smoke_test.py
Normal file
246
tools/stt_gateway_smoke_test.py
Normal file
@@ -0,0 +1,246 @@
|
||||
import asyncio
|
||||
import json
|
||||
import wave
|
||||
|
||||
import websockets
|
||||
|
||||
|
||||
GATEWAY_URL = "ws://127.0.0.1:8000/ws/audio"
|
||||
|
||||
AUDIO_FILE = (
|
||||
"/home/ubuntu/voice-assistant/audios/chafik_normalize.wav"
|
||||
)
|
||||
|
||||
CHUNK_FRAMES = 1600
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
print("========================================")
|
||||
print("STT GATEWAY SMOKE TEST")
|
||||
print("========================================")
|
||||
print(f"Gateway : {GATEWAY_URL}")
|
||||
print(f"Audio : {AUDIO_FILE}")
|
||||
print(f"Chunk : {CHUNK_FRAMES} frames")
|
||||
print()
|
||||
|
||||
async with websockets.connect(GATEWAY_URL) as websocket:
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# 1. session.start
|
||||
# -------------------------------------------------------------
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.start",
|
||||
"audio": {
|
||||
"encoding": "pcm_s16le",
|
||||
"sample_rate": 16000,
|
||||
"channels": 1,
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
response = json.loads(
|
||||
await websocket.recv()
|
||||
)
|
||||
|
||||
print(
|
||||
f"[RECV] {response['type']}"
|
||||
)
|
||||
|
||||
if response["type"] != "session.ready":
|
||||
raise RuntimeError(
|
||||
f"Expected session.ready, got {response}"
|
||||
)
|
||||
|
||||
session_id = response["session_id"]
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# 2. audio.start
|
||||
# -------------------------------------------------------------
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "audio.start",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
response = json.loads(
|
||||
await websocket.recv()
|
||||
)
|
||||
|
||||
print(
|
||||
f"[RECV] {response['type']}"
|
||||
)
|
||||
|
||||
if response["type"] != "turn.started":
|
||||
raise RuntimeError(
|
||||
f"Expected turn.started, got {response}"
|
||||
)
|
||||
|
||||
turn_id = response["turn_id"]
|
||||
|
||||
print(
|
||||
f"Session : {session_id}"
|
||||
)
|
||||
print(
|
||||
f"Turn : {turn_id}"
|
||||
)
|
||||
print()
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# 3. Start a receiver task.
|
||||
#
|
||||
# This is necessary because partial/final events can arrive
|
||||
# while we are still sending audio.
|
||||
# -------------------------------------------------------------
|
||||
received_events: list[dict] = []
|
||||
|
||||
async def receive_events() -> None:
|
||||
while True:
|
||||
message = await websocket.recv()
|
||||
event = json.loads(message)
|
||||
|
||||
received_events.append(event)
|
||||
|
||||
event_type = event.get("type")
|
||||
|
||||
if event_type == "stt.partial":
|
||||
print(
|
||||
f"[PARTIAL] {event.get('text', '')}"
|
||||
)
|
||||
|
||||
elif event_type == "stt.final":
|
||||
print(
|
||||
f"[FINAL ] {event.get('text', '')}"
|
||||
)
|
||||
|
||||
elif event_type == "turn.ended":
|
||||
print(
|
||||
"[RECV] turn.ended"
|
||||
)
|
||||
break
|
||||
|
||||
elif event_type == "error":
|
||||
print(
|
||||
"[ERROR]",
|
||||
event,
|
||||
)
|
||||
|
||||
break
|
||||
|
||||
else:
|
||||
print(
|
||||
f"[RECV] {event_type}"
|
||||
)
|
||||
|
||||
receiver_task = asyncio.create_task(
|
||||
receive_events()
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# 4. Stream WAV as:
|
||||
#
|
||||
# audio.chunk JSON
|
||||
# binary PCM
|
||||
#
|
||||
# The sequence number belongs to the logical chunk.
|
||||
# -------------------------------------------------------------
|
||||
with wave.open(AUDIO_FILE, "rb") as wav:
|
||||
sequence_number = 0
|
||||
|
||||
while True:
|
||||
audio = wav.readframes(CHUNK_FRAMES)
|
||||
|
||||
if not audio:
|
||||
break
|
||||
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "audio.chunk",
|
||||
"turn_id": turn_id,
|
||||
"sequence_number": sequence_number,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
await websocket.send(audio)
|
||||
|
||||
sequence_number += 1
|
||||
|
||||
print()
|
||||
print(
|
||||
f"Audio sent: {sequence_number} chunks"
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# 5. audio.stop
|
||||
# -------------------------------------------------------------
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "audio.stop",
|
||||
"turn_id": turn_id,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# 6. Wait for:
|
||||
#
|
||||
# stt.final
|
||||
# turn.ended
|
||||
#
|
||||
# The receiver task handles both.
|
||||
# -------------------------------------------------------------
|
||||
await receiver_task
|
||||
|
||||
print()
|
||||
print("========================================")
|
||||
print("RESULT")
|
||||
print("========================================")
|
||||
|
||||
partials = [
|
||||
event
|
||||
for event in received_events
|
||||
if event.get("type") == "stt.partial"
|
||||
]
|
||||
|
||||
finals = [
|
||||
event
|
||||
for event in received_events
|
||||
if event.get("type") == "stt.final"
|
||||
]
|
||||
|
||||
turn_ended = [
|
||||
event
|
||||
for event in received_events
|
||||
if event.get("type") == "turn.ended"
|
||||
]
|
||||
|
||||
print(
|
||||
f"Partial events : {len(partials)}"
|
||||
)
|
||||
print(
|
||||
f"Final events : {len(finals)}"
|
||||
)
|
||||
print(
|
||||
f"Turn ended : {len(turn_ended)}"
|
||||
)
|
||||
|
||||
if finals:
|
||||
print()
|
||||
print("Final transcript:")
|
||||
print(
|
||||
finals[-1].get("text", "")
|
||||
)
|
||||
|
||||
print()
|
||||
print("STT Gateway smoke test finished.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
114
tools/stt_smoke_test.py
Normal file
114
tools/stt_smoke_test.py
Normal file
@@ -0,0 +1,114 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from riva.client.asr import AudioChunkFileIterator
|
||||
from riva.client.realtime import RealtimeClientASR
|
||||
|
||||
|
||||
NIM_SERVER = "127.0.0.1:9000"
|
||||
INPUT_FILE = "/home/ubuntu/voice-assistant/audios/chafik_normalize.wav"
|
||||
CHUNK_FRAMES = 1600
|
||||
|
||||
|
||||
def build_args() -> argparse.Namespace:
|
||||
return argparse.Namespace(
|
||||
input_file=INPUT_FILE,
|
||||
mic=False,
|
||||
list_devices=False,
|
||||
duration=None,
|
||||
input_device=None,
|
||||
|
||||
sample_rate_hz=16000,
|
||||
num_channels=1,
|
||||
file_streaming_chunk=CHUNK_FRAMES,
|
||||
|
||||
output_text=None,
|
||||
prompt="",
|
||||
|
||||
server=NIM_SERVER,
|
||||
ssl_root_cert=None,
|
||||
ssl_client_cert=None,
|
||||
ssl_client_key=None,
|
||||
use_ssl=False,
|
||||
metadata=None,
|
||||
options=None,
|
||||
max_message_length=67108864,
|
||||
|
||||
word_time_offsets=False,
|
||||
max_alternatives=1,
|
||||
profanity_filter=False,
|
||||
automatic_punctuation=True,
|
||||
no_verbatim_transcripts=False,
|
||||
|
||||
language_code="pt-BR",
|
||||
model_name="",
|
||||
|
||||
boosted_lm_words=None,
|
||||
boosted_lm_score=4.0,
|
||||
|
||||
speaker_diarization=False,
|
||||
diarization_max_speakers=3,
|
||||
|
||||
start_history=-1,
|
||||
start_threshold=-1.0,
|
||||
stop_history=-1,
|
||||
stop_threshold=-1.0,
|
||||
stop_history_eou=-1,
|
||||
stop_threshold_eou=-1.0,
|
||||
|
||||
custom_configuration="",
|
||||
endpoint="/v1/realtime",
|
||||
query_params="intent=transcription",
|
||||
mint_api_key=None,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
input_path = Path(INPUT_FILE)
|
||||
|
||||
if not input_path.exists():
|
||||
raise FileNotFoundError(
|
||||
f"Audio file not found: {input_path}"
|
||||
)
|
||||
|
||||
args = build_args()
|
||||
|
||||
print(f"NIM server : {args.server}")
|
||||
print(f"Audio : {input_path}")
|
||||
print(f"Language : {args.language_code}")
|
||||
print(f"Chunk : {CHUNK_FRAMES} frames")
|
||||
|
||||
client = RealtimeClientASR(args=args)
|
||||
|
||||
audio_iterator = AudioChunkFileIterator(
|
||||
input_file=str(input_path),
|
||||
chunk_n_frames=CHUNK_FRAMES,
|
||||
delay_callback=None,
|
||||
)
|
||||
|
||||
try:
|
||||
print("Connecting to NIM...")
|
||||
await client.connect()
|
||||
print("Connected.")
|
||||
|
||||
send_task = asyncio.create_task(
|
||||
client.send_audio_chunks(audio_iterator)
|
||||
)
|
||||
|
||||
receive_task = asyncio.create_task(
|
||||
client.receive_responses()
|
||||
)
|
||||
|
||||
await asyncio.gather(
|
||||
send_task,
|
||||
receive_task,
|
||||
)
|
||||
|
||||
finally:
|
||||
await client.disconnect()
|
||||
print("Disconnected.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
0
tools/tts_gateway_smoke_test.py
Normal file
0
tools/tts_gateway_smoke_test.py
Normal file
327
tools/voice_pipeline_smoke_test.py
Normal file
327
tools/voice_pipeline_smoke_test.py
Normal file
@@ -0,0 +1,327 @@
|
||||
import asyncio
|
||||
import json
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
import websockets
|
||||
|
||||
|
||||
GATEWAY_URL = "ws://127.0.0.1:8001/ws/audio"
|
||||
|
||||
AUDIO_FILE = Path(
|
||||
"/home/ubuntu/voice-assistant/audios/chafik_normalize.wav"
|
||||
)
|
||||
|
||||
OUTPUT_FILE = Path(
|
||||
"/tmp/voice_pipeline_response.pcm"
|
||||
)
|
||||
|
||||
CHUNK_FRAMES = 1600
|
||||
|
||||
|
||||
async def receive_messages(
|
||||
websocket,
|
||||
state: dict,
|
||||
) -> None:
|
||||
while True:
|
||||
message = await websocket.recv()
|
||||
|
||||
if isinstance(message, bytes):
|
||||
state["tts_chunks"] += 1
|
||||
state["tts_bytes"] += len(message)
|
||||
state["tts_audio"].extend(message)
|
||||
|
||||
print(
|
||||
f"[TTS AUDIO] "
|
||||
f"chunk={state['tts_chunks']} "
|
||||
f"bytes={len(message)}"
|
||||
)
|
||||
|
||||
continue
|
||||
|
||||
event = json.loads(message)
|
||||
event_type = event.get("type")
|
||||
|
||||
if event_type == "stt.partial":
|
||||
state["partial_count"] += 1
|
||||
print(f"[PARTIAL] {event.get('text', '')}")
|
||||
|
||||
elif event_type == "stt.final":
|
||||
state["final_text"] = event.get("text")
|
||||
print(f"[FINAL ] {state['final_text']}")
|
||||
|
||||
elif event_type == "agent.response":
|
||||
state["agent_response"] = event.get("text")
|
||||
print(
|
||||
f"[AGENT ] "
|
||||
f"{state['agent_response']}"
|
||||
)
|
||||
|
||||
elif event_type == "turn.ended":
|
||||
state["turn_ended"] = True
|
||||
print("[RECV] turn.ended")
|
||||
return
|
||||
|
||||
elif event_type == "error":
|
||||
raise RuntimeError(
|
||||
f"Gateway error: {event}"
|
||||
)
|
||||
|
||||
else:
|
||||
print(f"[RECV] {event_type}")
|
||||
|
||||
|
||||
async def send_audio(
|
||||
websocket,
|
||||
turn_id: int,
|
||||
) -> int:
|
||||
sent_chunks = 0
|
||||
|
||||
with wave.open(str(AUDIO_FILE), "rb") as audio:
|
||||
sample_rate = audio.getframerate()
|
||||
channels = audio.getnchannels()
|
||||
sample_width = audio.getsampwidth()
|
||||
|
||||
if sample_rate != 16000:
|
||||
raise RuntimeError(
|
||||
f"Expected 16 kHz audio, got {sample_rate} Hz."
|
||||
)
|
||||
|
||||
if channels != 1:
|
||||
raise RuntimeError(
|
||||
f"Expected mono audio, got {channels} channels."
|
||||
)
|
||||
|
||||
if sample_width != 2:
|
||||
raise RuntimeError(
|
||||
f"Expected 16-bit PCM audio, got "
|
||||
f"{sample_width * 8}-bit."
|
||||
)
|
||||
|
||||
while True:
|
||||
pcm = audio.readframes(CHUNK_FRAMES)
|
||||
|
||||
if not pcm:
|
||||
break
|
||||
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "audio.chunk",
|
||||
"turn_id": turn_id,
|
||||
"sequence_number": sent_chunks,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
await websocket.send(pcm)
|
||||
|
||||
sent_chunks += 1
|
||||
|
||||
return sent_chunks
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
print("=" * 40)
|
||||
print("VOICE PIPELINE SMOKE TEST")
|
||||
print("=" * 40)
|
||||
print(f"Gateway : {GATEWAY_URL}")
|
||||
print(f"Audio : {AUDIO_FILE}")
|
||||
print(f"Chunk : {CHUNK_FRAMES} frames")
|
||||
print()
|
||||
|
||||
state = {
|
||||
"partial_count": 0,
|
||||
"final_text": None,
|
||||
"agent_response": None,
|
||||
"tts_chunks": 0,
|
||||
"tts_bytes": 0,
|
||||
"tts_audio": bytearray(),
|
||||
"turn_ended": False,
|
||||
}
|
||||
|
||||
async with websockets.connect(GATEWAY_URL) as websocket:
|
||||
# ---------------------------------------------------------
|
||||
# session.start
|
||||
# ---------------------------------------------------------
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.start",
|
||||
"audio": {
|
||||
"encoding": "pcm_s16le",
|
||||
"sample_rate": 16000,
|
||||
"channels": 1,
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
message = await websocket.recv()
|
||||
|
||||
if isinstance(message, bytes):
|
||||
raise RuntimeError(
|
||||
"Expected JSON session.ready."
|
||||
)
|
||||
|
||||
session_ready = json.loads(message)
|
||||
|
||||
print(
|
||||
f"[RECV] "
|
||||
f"{session_ready.get('type')}"
|
||||
)
|
||||
|
||||
if session_ready.get("type") != "session.ready":
|
||||
raise RuntimeError(
|
||||
f"Expected session.ready, got "
|
||||
f"{session_ready}"
|
||||
)
|
||||
|
||||
session_id = session_ready["session_id"]
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# audio.start
|
||||
# ---------------------------------------------------------
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "audio.start",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
message = await websocket.recv()
|
||||
|
||||
if isinstance(message, bytes):
|
||||
raise RuntimeError(
|
||||
"Expected JSON turn.started."
|
||||
)
|
||||
|
||||
turn_started = json.loads(message)
|
||||
|
||||
print(
|
||||
f"[RECV] "
|
||||
f"{turn_started.get('type')}"
|
||||
)
|
||||
|
||||
if turn_started.get("type") != "turn.started":
|
||||
raise RuntimeError(
|
||||
f"Expected turn.started, got "
|
||||
f"{turn_started}"
|
||||
)
|
||||
|
||||
turn_id = turn_started["turn_id"]
|
||||
|
||||
print()
|
||||
print(f"Session : {session_id}")
|
||||
print(f"Turn : {turn_id}")
|
||||
print()
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Run sender and receiver concurrently
|
||||
# ---------------------------------------------------------
|
||||
receiver_task = asyncio.create_task(
|
||||
receive_messages(
|
||||
websocket,
|
||||
state,
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
sent_chunks = await send_audio(
|
||||
websocket,
|
||||
turn_id,
|
||||
)
|
||||
except Exception:
|
||||
receiver_task.cancel()
|
||||
raise
|
||||
|
||||
print()
|
||||
print(
|
||||
f"Audio sent: {sent_chunks} chunks"
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Tell Gateway that user audio is complete.
|
||||
# ---------------------------------------------------------
|
||||
await websocket.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "audio.stop",
|
||||
"turn_id": turn_id,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Wait for the entire pipeline to finish.
|
||||
# ---------------------------------------------------------
|
||||
await asyncio.wait_for(
|
||||
receiver_task,
|
||||
timeout=120.0,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# Validate the complete STT -> Agent -> TTS pipeline.
|
||||
# ---------------------------------------------------------
|
||||
if not state["turn_ended"]:
|
||||
raise RuntimeError("Turn did not complete.")
|
||||
|
||||
if not state["final_text"]:
|
||||
raise RuntimeError("STT final transcript was not received.")
|
||||
|
||||
if not state["agent_response"]:
|
||||
raise RuntimeError("Agent response was not received.")
|
||||
|
||||
if state["tts_chunks"] == 0:
|
||||
raise RuntimeError("No TTS audio was received.")
|
||||
|
||||
if state["tts_bytes"] == 0:
|
||||
raise RuntimeError("TTS output is empty.")
|
||||
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# Save TTS audio
|
||||
# -------------------------------------------------------------
|
||||
OUTPUT_FILE.write_bytes(
|
||||
bytes(state["tts_audio"])
|
||||
)
|
||||
|
||||
print()
|
||||
print("=" * 40)
|
||||
print("RESULT")
|
||||
print("=" * 40)
|
||||
print(f"Session : {session_id}")
|
||||
print(f"Turn : {turn_id}")
|
||||
print(
|
||||
f"Audio sent : {sent_chunks}"
|
||||
)
|
||||
print(
|
||||
f"Partial events : {state['partial_count']}"
|
||||
)
|
||||
print(
|
||||
f"Final transcript: {state['final_text']}"
|
||||
)
|
||||
print(
|
||||
f"Agent response : {state['agent_response']}"
|
||||
)
|
||||
print(
|
||||
f"TTS chunks : {state['tts_chunks']}"
|
||||
)
|
||||
print(
|
||||
f"TTS bytes : {state['tts_bytes']}"
|
||||
)
|
||||
print(
|
||||
f"TTS output : {OUTPUT_FILE}"
|
||||
)
|
||||
print(
|
||||
f"Turn ended : {state['turn_ended']}"
|
||||
)
|
||||
print()
|
||||
print(
|
||||
"Voice pipeline smoke test finished."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user