refactor: formaliza formatos de áudio e alinhamento PCM

This commit is contained in:
alex alves
2026-11-03 16:43:22 -03:00
parent 4d7ff46a36
commit 76d15b8e52
23 changed files with 349 additions and 82 deletions

View File

@@ -311,6 +311,7 @@ The `staging` and `production` files are present, but deployment readiness for t
| Endpoint | `ws://127.0.0.1:8001/ws/audio` | | Endpoint | `ws://127.0.0.1:8001/ws/audio` |
| Protocol version | `1.0` | | Protocol version | `1.0` |
| Input audio | PCM signed 16-bit little-endian, 16 kHz, mono | | Input audio | PCM signed 16-bit little-endian, 16 kHz, mono |
| Output audio | Declared by `session.ready.output_audio`; current VoxCPM: PCM S16LE, 48 kHz, mono |
| Transport: | JSON control messages and binary PCM frames. | | Transport: | JSON control messages and binary PCM frames. |
### Typical exchange ### Typical exchange
@@ -340,7 +341,7 @@ sequenceDiagram
G-->>C: turn.ended G-->>C: turn.ended
``` ```
STT events may arrive while input audio is still streaming. Binary audio input is preceded by an `audio.chunk` control message containing `turn_id` and `sequence_number`. The Gateway limits each binary chunk to 64 KiB. STT events may arrive while input audio is still streaming. Binary audio input is preceded by an `audio.chunk` control message containing `turn_id` and `sequence_number`. The Gateway limits each binary chunk to 64 KiB. Input frames must contain complete PCM samples; rejected frames do not advance the audio sequence. `session.ready.audio` describes input and `session.ready.output_audio` describes output. The browser configures playback from the declared output format. VoxCPM reassembles HTTP fragments into complete PCM frames and rejects a truncated final sample. Its `sample_rate` option declares the expected native output rate; it does not request resampling from the external service.
### Interruption and session closure ### Interruption and session closure

View File

@@ -9,6 +9,7 @@ tts:
provider: voxcpm provider: voxcpm
options: options:
server: "http://127.0.0.1:8000" server: "http://127.0.0.1:8000"
sample_rate: 48000
model: "openbmb/VoxCPM2" model: "openbmb/VoxCPM2"
voice: "default" voice: "default"

View File

@@ -2,6 +2,8 @@
from dataclasses import dataclass from dataclasses import dataclass
from voice_assistant.audio.models import AudioFormat
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
class TranscriptPartial: class TranscriptPartial:
@@ -36,6 +38,7 @@ class SpeechChunk:
session_id: str session_id: str
turn_id: int turn_id: int
audio: bytes audio: bytes
audio_format: AudioFormat
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)

View File

@@ -16,6 +16,7 @@ from voice_assistant.application.events import (
TurnCompleted, TurnCompleted,
VoiceEvent, VoiceEvent,
) )
from voice_assistant.audio.models import AudioFormat, CANONICAL_AUDIO_FORMAT
from voice_assistant.stt.adapter import STTAdapter from voice_assistant.stt.adapter import STTAdapter
from voice_assistant.voice_pipeline import ( from voice_assistant.voice_pipeline import (
PipelineAgentResponse, PipelineAgentResponse,
@@ -95,6 +96,10 @@ class VoiceSession:
def closed(self) -> bool: def closed(self) -> bool:
return self._pipeline.session.closed return self._pipeline.session.closed
@property
def output_audio_format(self) -> AudioFormat:
return self._pipeline.tts.output_format
def validate_audio_input(self, turn_id: int) -> None: def validate_audio_input(self, turn_id: int) -> None:
self._require_open() self._require_open()
if self._audio_input_closed: if self._audio_input_closed:
@@ -165,6 +170,7 @@ class VoiceSession:
self, turn_id: int, sequence_number: int, audio: bytes, self, turn_id: int, sequence_number: int, audio: bytes,
) -> None: ) -> None:
self.validate_audio_input(turn_id) self.validate_audio_input(turn_id)
CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
self._pipeline.session.accept_audio_chunk( self._pipeline.session.accept_audio_chunk(
turn_id, sequence_number, len(audio), turn_id, sequence_number, len(audio),
) )
@@ -258,7 +264,7 @@ class VoiceSession:
elif isinstance(event, PipelineAgentResponse): elif isinstance(event, PipelineAgentResponse):
await self._emit(AgentReply(self.session_id, turn_id, event.text)) await self._emit(AgentReply(self.session_id, turn_id, event.text))
elif isinstance(event, PipelineAudioChunk): elif isinstance(event, PipelineAudioChunk):
await self._emit(SpeechChunk(self.session_id, turn_id, event.audio)) await self._emit(SpeechChunk(self.session_id, turn_id, event.audio, self.output_audio_format))
else: else:
result = await self._pipeline.process_transcript( result = await self._pipeline.process_transcript(
session_id=self.session_id, turn_id=turn_id, text=text, session_id=self.session_id, turn_id=turn_id, text=text,
@@ -268,7 +274,7 @@ class VoiceSession:
self._pipeline.synthesize_response(result.response_text) self._pipeline.synthesize_response(result.response_text)
) as audio_stream: ) as audio_stream:
async for audio in audio_stream: async for audio in audio_stream:
await self._emit(SpeechChunk(self.session_id, turn_id, audio)) await self._emit(SpeechChunk(self.session_id, turn_id, audio, self.output_audio_format))
self._pipeline.on_tts_complete(turn_id) self._pipeline.on_tts_complete(turn_id)
async def _finalize_turn(self, turn_id: int, stt: STTAdapter) -> None: async def _finalize_turn(self, turn_id: int, stt: STTAdapter) -> None:

View File

@@ -1,58 +1,36 @@
""" """PCM format contracts shared by application, transports, and providers.
Canonical audio models for the Voice Assistant.
Purpose: The canonical input is mono PCM S16LE at 16 kHz. Output uses the format
Define the provider-independent audio format used internally by explicitly declared by the TTS provider. Declaring a sample rate does not
the Voice Core. convert or resample audio; provider adapters own any required conversion.
Responsibilities:
- Describe an audio format in a typed structure.
- Define the canonical audio format used by the Voice Core.
- Provide a shared audio contract for transports and providers.
Non-responsibilities:
- Encoding or decoding audio.
- Resampling audio.
- Converting provider-specific audio formats.
- Managing WebSocket audio frames.
- Performing STT or TTS operations.
Architecture:
Client / Transport
|
v
Canonical Audio
PCM S16LE / 16 kHz / mono
|
v
Voice Core
|
+-----+-----+
| |
v v
STT TTS
Adapter Adapter
Provider-specific format conversion, when required, belongs
inside the corresponding provider adapter.
""" """
from typing import Literal from typing import Literal
from pydantic import BaseModel from pydantic import BaseModel, ConfigDict, Field
class AudioFormat(BaseModel): class AudioFormat(BaseModel):
""" """An immutable PCM format; a frame contains one sample per channel."""
Describe one PCM audio format.
This model is independent from WebSocket, STT, TTS, or any model_config = ConfigDict(frozen=True)
concrete provider.
"""
encoding: Literal["pcm_s16le"] encoding: Literal["pcm_s16le"]
sample_rate: int sample_rate: int = Field(gt=0, strict=True)
channels: int channels: int = Field(gt=0, strict=True)
@property
def frame_size_bytes(self) -> int:
return 2 * self.channels
def validate_chunk(self, audio: bytes) -> None:
if not audio:
raise ValueError("Audio chunk cannot be empty.")
if len(audio) % self.frame_size_bytes:
raise ValueError(
"PCM chunk must contain complete frames of "
f"{self.frame_size_bytes} bytes."
)
CANONICAL_AUDIO_FORMAT = AudioFormat( CANONICAL_AUDIO_FORMAT = AudioFormat(
@@ -60,4 +38,3 @@ CANONICAL_AUDIO_FORMAT = AudioFormat(
sample_rate=16000, sample_rate=16000,
channels=1, channels=1,
) )

View File

@@ -226,6 +226,7 @@ async def audio_socket(websocket: WebSocket) -> None:
await send_json(websocket, SessionReady( await send_json(websocket, SessionReady(
protocol_version=PROTOCOL_VERSION, session_id=session.session_id, protocol_version=PROTOCOL_VERSION, session_id=session.session_id,
audio=CANONICAL_AUDIO_FORMAT, audio=CANONICAL_AUDIO_FORMAT,
output_audio=session.output_audio_format,
)) ))
continue continue
if session is None: if session is None:

View File

@@ -10,9 +10,9 @@ from voice_assistant.audio.models import (
) )
PROTOCOL_VERSION = "1.0" PROTOCOL_VERSION = "1.0"
AUDIO_ENCODING = "pcm_s16le" AUDIO_ENCODING = CANONICAL_AUDIO_FORMAT.encoding
AUDIO_SAMPLE_RATE = 16000 AUDIO_SAMPLE_RATE = CANONICAL_AUDIO_FORMAT.sample_rate
AUDIO_CHANNELS = 1 AUDIO_CHANNELS = CANONICAL_AUDIO_FORMAT.channels
class SessionStart(BaseModel): class SessionStart(BaseModel):
@@ -81,6 +81,7 @@ class SessionReady(BaseModel):
protocol_version: str protocol_version: str
session_id: str session_id: str
audio: AudioFormat audio: AudioFormat
output_audio: AudioFormat
class TurnStarted(BaseModel): class TurnStarted(BaseModel):

View File

@@ -62,6 +62,22 @@ export class PCMPlayer {
return this.analyser; return this.analyser;
} }
configure(format) {
if (
format?.encoding !== "pcm_s16le" ||
format.channels !== 1 ||
!Number.isInteger(format.sample_rate) ||
format.sample_rate < 8000 ||
format.sample_rate > 96000
) {
throw new Error("Formato de áudio de saída incompatível.");
}
if (this.sources.size > 0) {
throw new Error("Não é possível trocar o formato durante a reprodução.");
}
this.sampleRate = format.sample_rate;
}
decodePCM(buffer) { decodePCM(buffer) {
if (!(buffer instanceof ArrayBuffer)) { if (!(buffer instanceof ArrayBuffer)) {
throw new TypeError("PCM chunk must be an ArrayBuffer."); throw new TypeError("PCM chunk must be an ArrayBuffer.");

View File

@@ -102,7 +102,6 @@ export class VoiceSessionController {
this.player = new PCMPlayer({ this.player = new PCMPlayer({
audioContext, audioContext,
sampleRate: 48000,
onPlaybackStart: () => { onPlaybackStart: () => {
if ( if (
@@ -547,6 +546,7 @@ export class VoiceSessionController {
handleGatewayEvent(event) { handleGatewayEvent(event) {
switch (event.type) { switch (event.type) {
case "session.ready": case "session.ready":
this.player.configure(event.output_audio);
this.startListening(); this.startListening();
break; break;

View File

@@ -34,6 +34,8 @@ Architecture:
import asyncio import asyncio
from collections.abc import AsyncIterator from collections.abc import AsyncIterator
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
from .adapter import STTAdapter from .adapter import STTAdapter
from .models import TranscriptEvent from .models import TranscriptEvent
@@ -56,10 +58,7 @@ class MockSTTAdapter(STTAdapter):
"STT session has not been started." "STT session has not been started."
) )
if not audio: CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
raise ValueError(
"Audio chunk cannot be empty."
)
async def stop_session(self) -> None: async def stop_session(self) -> None:
if not self._started: if not self._started:

View File

@@ -105,8 +105,7 @@ class NvidiaSTTAdapter(STTAdapter):
if not self._started or self._websocket is None: if not self._started or self._websocket is None:
raise RuntimeError("STT session has not been started.") raise RuntimeError("STT session has not been started.")
if not audio: CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
raise ValueError("Audio chunk cannot be empty.")
await self._send_json( await self._send_json(
{ {

View File

@@ -3,13 +3,22 @@
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import AsyncGenerator from collections.abc import AsyncGenerator
from voice_assistant.audio.models import AudioFormat
class TTSAdapter(ABC): class TTSAdapter(ABC):
@property
@abstractmethod
def output_format(self) -> AudioFormat:
"""Declare the actual PCM format emitted throughout the session."""
raise NotImplementedError
@abstractmethod @abstractmethod
def synthesize(self, text: str) -> AsyncGenerator[bytes, None]: def synthesize(self, text: str) -> AsyncGenerator[bytes, None]:
"""Return an audio generator consumed directly with async for. """Return an audio generator consumed directly with async for.
The caller closes it on early exit. Implementations must release Each yielded chunk must contain nonempty, complete PCM frames
matching output_format. The caller closes it on early exit. Implementations must release
stream-owned resources in finally blocks and propagate cancellation stream-owned resources in finally blocks and propagate cancellation
and provider errors. The caller does not await synthesize itself. and provider errors. The caller does not await synthesize itself.
""" """

View File

@@ -1,9 +1,15 @@
from collections.abc import AsyncGenerator from collections.abc import AsyncGenerator
from voice_assistant.audio.models import AudioFormat
from .adapter import TTSAdapter from .adapter import TTSAdapter
class MockTTSAdapter(TTSAdapter): class MockTTSAdapter(TTSAdapter):
output_format = AudioFormat(
encoding="pcm_s16le", sample_rate=48000, channels=1,
)
def __init__(self) -> None: def __init__(self) -> None:
self.calls: list[str] = [] self.calls: list[str] = []

View File

@@ -2,6 +2,8 @@ from collections.abc import AsyncGenerator
import httpx import httpx
from voice_assistant.audio.models import AudioFormat
from .adapter import TTSAdapter from .adapter import TTSAdapter
@@ -15,7 +17,15 @@ class VoxCPMTTSAdapter(TTSAdapter):
stream: bool = True, stream: bool = True,
stream_format: str = "audio", stream_format: str = "audio",
timeout: float = 120.0, timeout: float = 120.0,
sample_rate: int = 48000,
) -> None: ) -> None:
if response_format != "pcm":
raise ValueError("VoxCPM must emit raw PCM audio.")
if stream_format != "audio":
raise ValueError("VoxCPM must emit audio bytes, not SSE events.")
self._output_format = AudioFormat(
encoding="pcm_s16le", sample_rate=sample_rate, channels=1,
)
self.server = server.rstrip("/") self.server = server.rstrip("/")
self.model = model self.model = model
self.voice = voice self.voice = voice
@@ -24,6 +34,10 @@ class VoxCPMTTSAdapter(TTSAdapter):
self.stream_format = stream_format self.stream_format = stream_format
self.timeout = timeout self.timeout = timeout
@property
def output_format(self) -> AudioFormat:
return self._output_format
async def synthesize( async def synthesize(
self, self,
text: str, text: str,
@@ -51,7 +65,17 @@ class VoxCPMTTSAdapter(TTSAdapter):
json=payload, json=payload,
) as response: ) as response:
response.raise_for_status() response.raise_for_status()
content_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if content_type not in {"audio/pcm", "application/octet-stream"}:
raise ValueError("TTS response must contain raw PCM audio.")
remainder = b""
frame_size = self.output_format.frame_size_bytes
async for chunk in response.aiter_bytes(): async for chunk in response.aiter_bytes():
if chunk: data = remainder + chunk
yield chunk aligned_size = len(data) - len(data) % frame_size
remainder = data[aligned_size:]
if aligned_size:
yield data[:aligned_size]
if remainder:
raise ValueError("TTS stream ended with an incomplete PCM frame.")

View File

@@ -183,6 +183,7 @@ class VoicePipeline:
""" """
async with aclosing(self.tts.synthesize(response_text)) as audio_stream: async with aclosing(self.tts.synthesize(response_text)) as audio_stream:
async for audio_chunk in audio_stream: async for audio_chunk in audio_stream:
self.tts.output_format.validate_chunk(audio_chunk)
yield audio_chunk yield audio_chunk
@@ -280,6 +281,7 @@ class VoicePipeline:
self.tts.synthesize(segment) self.tts.synthesize(segment)
) as audio_stream: ) as audio_stream:
async for audio_chunk in audio_stream: async for audio_chunk in audio_stream:
self.tts.output_format.validate_chunk(audio_chunk)
await events.put( await events.put(
PipelineAudioChunk(audio=audio_chunk) PipelineAudioChunk(audio=audio_chunk)
) )

View File

@@ -223,3 +223,35 @@ test("late PCM after stop is ignored", async () => {
assert.equal(player.playing, false); assert.equal(player.playing, false);
assert.equal(player.sources.size, 0); assert.equal(player.sources.size, 0);
}); });
test("uses the output sample rate declared by the server", async () => {
const context = new FakeAudioContext();
const player = new PCMPlayer({ audioContext: context });
player.configure({ encoding: "pcm_s16le", sample_rate: 24000, channels: 1 });
await player.enqueue(pcm([100, 200, 300, 400]));
assert.equal(player.sampleRate, 24000);
assert.equal(context.sources[0].buffer.duration, 4 / 24000);
});
for (const format of [
undefined,
{ encoding: "wav", sample_rate: 48000, channels: 1 },
{ encoding: "pcm_s16le", sample_rate: 48000, channels: 2 },
{ encoding: "pcm_s16le", sample_rate: "48000", channels: 1 },
{ encoding: "pcm_s16le", sample_rate: 0, channels: 1 },
{ encoding: "pcm_s16le", sample_rate: 192000, channels: 1 },
]) {
test(`rejects unsupported playback metadata: ${JSON.stringify(format)}`, () => {
const player = new PCMPlayer({ audioContext: new FakeAudioContext() });
assert.throws(() => player.configure(format), /Formato de áudio/);
});
}
test("rejects format changes while audio is scheduled", async () => {
const player = new PCMPlayer({ audioContext: new FakeAudioContext() });
await player.enqueue(pcm([100, 200]));
assert.throws(() => player.configure({
encoding: "pcm_s16le", sample_rate: 24000, channels: 1,
}), /durante a reprodução/);
});

View File

@@ -733,3 +733,14 @@ test("barge-in resets player before new TTS response", () => {
assert.equal(finished, false); assert.equal(finished, false);
assert.equal(resetCount, 1); assert.equal(resetCount, 1);
}); });
test("session.ready configures output before starting microphone capture", () => {
const { controller } = makeController();
const operations = [];
const format = { encoding: "pcm_s16le", sample_rate: 24000, channels: 1 };
controller.player = { configure: (received) => operations.push(received) };
controller.startListening = () => operations.push("listen");
controller.handleGatewayEvent({ type: "session.ready", output_audio: format });
assert.deepEqual(operations, [format, "listen"]);
});

View File

@@ -0,0 +1,175 @@
"""PCM framing, per-direction metadata, and HTTP fragmentation guarantees."""
from contextlib import aclosing
import httpx
import pytest
from fastapi.testclient import TestClient
from pydantic import ValidationError
from voice_assistant.agent.mock import MockAgentAdapter
from voice_assistant.application.voice_session import VoiceSession
from voice_assistant.audio.models import AudioFormat, CANONICAL_AUDIO_FORMAT
from voice_assistant.audio_gateway.app import main
from voice_assistant.domain.session import SessionState
from voice_assistant.stt.mock import MockSTTAdapter
from voice_assistant.tts.mock import MockTTSAdapter
from voice_assistant.tts.voxcpm import VoxCPMTTSAdapter
from voice_assistant.turn_manager import TurnManager
from voice_assistant.voice_pipeline import VoicePipeline
@pytest.fixture
def anyio_backend():
return "asyncio"
@pytest.mark.parametrize("field", ["sample_rate", "channels"])
@pytest.mark.parametrize("value", [0, -1, True, "16000", 1.5])
def test_audio_format_requires_positive_integer_dimensions(field, value):
fields = {"encoding": "pcm_s16le", "sample_rate": 16000, "channels": 1}
fields[field] = value
with pytest.raises(ValidationError):
AudioFormat(**fields)
def test_shared_format_cannot_be_mutated():
with pytest.raises(ValidationError):
CANONICAL_AUDIO_FORMAT.sample_rate = 48000
assert CANONICAL_AUDIO_FORMAT.sample_rate == 16000
@pytest.mark.parametrize("channels", [1, 2])
@pytest.mark.parametrize("size", [0, 1, 2, 3, 4, 5, 8])
def test_pcm_validation_requires_complete_channel_frames(channels, size):
format_ = AudioFormat(encoding="pcm_s16le", sample_rate=16000, channels=channels)
if not size or size % (2 * channels):
with pytest.raises(ValueError):
format_.validate_chunk(bytes(size))
else:
format_.validate_chunk(bytes(size))
def make_pipeline(tts):
return VoicePipeline(
session=SessionState(), turn_manager=TurnManager(),
agent=MockAgentAdapter(), tts=tts,
)
@pytest.mark.anyio
async def test_invalid_input_does_not_advance_sequence_or_reach_stt():
class STT(MockSTTAdapter):
def __init__(self):
super().__init__()
self.received = []
async def send_audio(self, audio):
await super().send_audio(audio)
self.received.append(audio)
async def emit(event):
pass
stt = STT()
session = VoiceSession(
pipeline=make_pipeline(MockTTSAdapter()), create_stt=lambda: stt, emit=emit,
)
try:
turn_id = await session.start_turn()
with pytest.raises(ValueError, match="complete frames"):
await session.send_audio(turn_id, 0, b"\x00")
assert session.expected_audio_sequence == 0
assert stt.received == []
await session.send_audio(turn_id, 0, b"\x00\x00")
assert session.expected_audio_sequence == 1
assert stt.received == [b"\x00\x00"]
finally:
await session.aclose()
@pytest.mark.anyio
async def test_pipeline_rejects_misaligned_tts_output():
class TTS(MockTTSAdapter):
async def synthesize(self, text):
yield b"\x00"
async with aclosing(make_pipeline(TTS()).synthesize_response("Olá")) as stream:
with pytest.raises(ValueError, match="complete frames"):
await anext(stream)
class FragmentedStream(httpx.AsyncByteStream):
def __init__(self, chunks):
self.chunks = chunks
async def __aiter__(self):
for chunk in self.chunks:
yield chunk
def mock_tts_http(monkeypatch, chunks, content_type="audio/pcm"):
original_client = httpx.AsyncClient
def handle(request):
return httpx.Response(
200, headers={"content-type": content_type},
stream=FragmentedStream(chunks),
)
monkeypatch.setattr(
httpx, "AsyncClient",
lambda **kwargs: original_client(transport=httpx.MockTransport(handle), **kwargs),
)
@pytest.mark.anyio
@pytest.mark.parametrize("chunks", [
[b"\x01", b"\x02\x03", b"", b"\x04"],
[b"\x01\x02\x03", b"\x04"],
[b"\x01\x02\x03\x04"],
])
async def test_voxcpm_reassembles_http_fragments_without_losing_bytes(monkeypatch, chunks):
mock_tts_http(monkeypatch, chunks)
tts = VoxCPMTTSAdapter(server="http://test", model="test", voice="default")
output = [chunk async for chunk in tts.synthesize("Olá")]
assert b"".join(output) == b"\x01\x02\x03\x04"
assert all(chunk and len(chunk) % 2 == 0 for chunk in output)
@pytest.mark.anyio
async def test_voxcpm_rejects_truncated_final_frame(monkeypatch):
mock_tts_http(monkeypatch, [b"\x01\x02\x03"])
tts = VoxCPMTTSAdapter(server="http://test", model="test", voice="default")
with pytest.raises(ValueError, match="incomplete PCM frame"):
_ = [chunk async for chunk in tts.synthesize("Olá")]
@pytest.mark.anyio
@pytest.mark.parametrize("content_type", ["audio/wav", "text/event-stream", "application/json"])
async def test_voxcpm_rejects_non_pcm_response(monkeypatch, content_type):
mock_tts_http(monkeypatch, [b"data"], content_type)
tts = VoxCPMTTSAdapter(server="http://test", model="test", voice="default")
with pytest.raises(ValueError, match="raw PCM"):
_ = [chunk async for chunk in tts.synthesize("Olá")]
@pytest.mark.parametrize("options", [{"response_format": "wav"}, {"stream_format": "sse"}])
def test_voxcpm_rejects_incompatible_transport_settings(options):
with pytest.raises(ValueError):
VoxCPMTTSAdapter(server="http://test", model="test", voice="default", **options)
def test_gateway_announces_provider_output_format_and_preserves_input(monkeypatch):
class TTS(MockTTSAdapter):
output_format = AudioFormat(encoding="pcm_s16le", sample_rate=24000, channels=1)
monkeypatch.setattr(main, "create_tts_adapter", lambda **kwargs: TTS())
monkeypatch.setattr(main, "create_agent_adapter", lambda **kwargs: MockAgentAdapter())
with TestClient(main.app).websocket_connect("/ws/audio") as socket:
socket.send_json({"type": "session.start"})
ready = socket.receive_json()
assert ready["audio"]["sample_rate"] == 16000
assert ready["output_audio"] == {
"encoding": "pcm_s16le", "sample_rate": 24000, "channels": 1,
}

View File

@@ -1,5 +1,4 @@
from voice_assistant.agent.adapter import StreamingAgentAdapter from voice_assistant.agent.adapter import StreamingAgentAdapter
from voice_assistant.tts.adapter import TTSAdapter
import asyncio import asyncio
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager from contextlib import contextmanager
@@ -1865,9 +1864,9 @@ def test_websocket_streaming_agent_and_tts():
yield "Olá, como posso ajudar?" yield "Olá, como posso ajudar?"
yield " Estou à disposição." yield " Estou à disposição."
class StreamingTestTTS(TTSAdapter): class StreamingTestTTS(MockTTSAdapter):
async def synthesize(self, text): async def synthesize(self, text):
yield text.encode("utf-8") yield text.encode("utf-16-le")
with patch.object( with patch.object(
main, main,
@@ -1967,7 +1966,7 @@ def test_websocket_interrupt_during_streaming_tts():
async def stream(self, session_id, turn_id, text): async def stream(self, session_id, turn_id, text):
yield "Olá, esta é uma resposta longa." yield "Olá, esta é uma resposta longa."
class SlowStreamingTTS(TTSAdapter): class SlowStreamingTTS(MockTTSAdapter):
async def synthesize(self, text): async def synthesize(self, text):
try: try:
tts_started.set() tts_started.set()
@@ -2106,7 +2105,7 @@ def test_websocket_session_end_during_streaming_tts():
async def stream(self, session_id, turn_id, text): async def stream(self, session_id, turn_id, text):
yield "Esta é uma resposta longa." yield "Esta é uma resposta longa."
class SlowStreamingTTS(TTSAdapter): class SlowStreamingTTS(MockTTSAdapter):
async def synthesize(self, text): async def synthesize(self, text):
try: try:
yield b"\x01\x02\x03\x04" yield b"\x01\x02\x03\x04"
@@ -2303,7 +2302,7 @@ def test_websocket_interrupt_wins_pending_tts_completion():
async def stream(self, session_id, turn_id, text): async def stream(self, session_id, turn_id, text):
yield "Esta resposta será interrompida." yield "Esta resposta será interrompida."
class ControlledTTS(TTSAdapter): class ControlledTTS(MockTTSAdapter):
async def synthesize(self, text): async def synthesize(self, text):
try: try:
yield b"\x01\x02\x03\x04" yield b"\x01\x02\x03\x04"

View File

@@ -18,7 +18,7 @@ Non-responsibilities:
import pytest import pytest
from pydantic import ValidationError from pydantic import ValidationError
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT from voice_assistant.audio.models import AudioFormat, CANONICAL_AUDIO_FORMAT
from voice_assistant.audio_gateway.app.protocol import ( from voice_assistant.audio_gateway.app.protocol import (
AgentDelta, AgentDelta,
AgentResponse, AgentResponse,
@@ -83,6 +83,7 @@ def test_session_ready_output():
protocol_version="1.0", protocol_version="1.0",
session_id="session-1", session_id="session-1",
audio=CANONICAL_AUDIO_FORMAT, audio=CANONICAL_AUDIO_FORMAT,
output_audio=AudioFormat(encoding="pcm_s16le", sample_rate=48000, channels=1),
) )
assert event.model_dump() == { assert event.model_dump() == {
@@ -94,6 +95,11 @@ def test_session_ready_output():
"sample_rate": 16000, "sample_rate": 16000,
"channels": 1, "channels": 1,
}, },
"output_audio": {
"encoding": "pcm_s16le",
"sample_rate": 48000,
"channels": 1,
},
} }

View File

@@ -12,7 +12,6 @@ from voice_assistant.application.events import SpeechChunk
from voice_assistant.application.voice_session import VoiceSession from voice_assistant.application.voice_session import VoiceSession
from voice_assistant.domain.session import SessionState from voice_assistant.domain.session import SessionState
from voice_assistant.stt.mock import MockSTTAdapter from voice_assistant.stt.mock import MockSTTAdapter
from voice_assistant.tts.adapter import TTSAdapter
from voice_assistant.tts.mock import MockTTSAdapter from voice_assistant.tts.mock import MockTTSAdapter
from voice_assistant.turn_manager import TurnManager from voice_assistant.turn_manager import TurnManager
from voice_assistant.voice_pipeline import VoicePipeline from voice_assistant.voice_pipeline import VoicePipeline
@@ -93,17 +92,17 @@ async def test_streaming_cancellation_releases_stream():
async def test_stopping_synthesis_early_closes_provider_generator(): async def test_stopping_synthesis_early_closes_provider_generator():
closed = asyncio.Event() closed = asyncio.Event()
class TTS(TTSAdapter): class TTS(MockTTSAdapter):
async def synthesize(self, text): async def synthesize(self, text):
try: try:
yield b"audio" yield b"audi"
await asyncio.Event().wait() await asyncio.Event().wait()
finally: finally:
closed.set() closed.set()
pipeline = make_pipeline(MockAgentAdapter(), TTS()) pipeline = make_pipeline(MockAgentAdapter(), TTS())
async with aclosing(pipeline.synthesize_response("Olá")) as stream: async with aclosing(pipeline.synthesize_response("Olá")) as stream:
assert await anext(stream) == b"audio" assert await anext(stream) == b"audi"
assert closed.is_set() assert closed.is_set()
@@ -113,10 +112,10 @@ async def test_interrupt_during_delivery_closes_nonstreaming_tts():
synthesis_closed = asyncio.Event() synthesis_closed = asyncio.Event()
stt = MockSTTAdapter() stt = MockSTTAdapter()
class TTS(TTSAdapter): class TTS(MockTTSAdapter):
async def synthesize(self, text): async def synthesize(self, text):
try: try:
yield b"audio" yield b"audi"
await asyncio.Event().wait() await asyncio.Event().wait()
finally: finally:
synthesis_closed.set() synthesis_closed.set()

View File

@@ -1,5 +1,4 @@
from voice_assistant.agent.adapter import StreamingAgentAdapter from voice_assistant.agent.adapter import StreamingAgentAdapter
from voice_assistant.tts.adapter import TTSAdapter
""" """
Tests for the Voice Pipeline lifecycle orchestration. Tests for the Voice Pipeline lifecycle orchestration.
@@ -850,9 +849,9 @@ async def test_stream_turn_response_produces_audio_before_agent_finishes():
yield " Estou à disposição." yield " Estou à disposição."
self.finished = True self.finished = True
class StreamingTTS(TTSAdapter): class StreamingTTS(MockTTSAdapter):
async def synthesize(self, text): async def synthesize(self, text):
yield text.encode("utf-8") yield text.encode("utf-16-le")
agent = StreamingAgent() agent = StreamingAgent()
@@ -938,9 +937,9 @@ async def test_stream_turn_response_propagates_agent_failure():
yield "Olá, como posso ajudar?" yield "Olá, como posso ajudar?"
raise RuntimeError("Simulated OCI streaming failure.") raise RuntimeError("Simulated OCI streaming failure.")
class StreamingTTS(TTSAdapter): class StreamingTTS(MockTTSAdapter):
async def synthesize(self, text): async def synthesize(self, text):
yield text.encode("utf-8") yield text.encode("utf-16-le")
pipeline = VoicePipeline( pipeline = VoicePipeline(
session=SessionState(session_id="agent-failure-session"), session=SessionState(session_id="agent-failure-session"),
@@ -984,7 +983,7 @@ async def test_stream_turn_response_cancels_agent_on_tts_failure():
finally: finally:
self.cancelled.set() self.cancelled.set()
class FailingTTS(TTSAdapter): class FailingTTS(MockTTSAdapter):
async def synthesize(self, text): async def synthesize(self, text):
raise RuntimeError("Simulated TTS failure.") raise RuntimeError("Simulated TTS failure.")
yield b"" # Keep this method an async generator. yield b"" # Keep this method an async generator.
@@ -1036,7 +1035,7 @@ async def test_stream_turn_response_external_cancellation():
finally: finally:
self.closed.set() self.closed.set()
class BlockingTTS(TTSAdapter): class BlockingTTS(MockTTSAdapter):
def __init__(self): def __init__(self):
self.started = asyncio.Event() self.started = asyncio.Event()
self.closed = asyncio.Event() self.closed = asyncio.Event()
@@ -1045,7 +1044,7 @@ async def test_stream_turn_response_external_cancellation():
try: try:
self.started.set() self.started.set()
await asyncio.Event().wait() await asyncio.Event().wait()
yield b"audio" yield b"audi"
finally: finally:
self.closed.set() self.closed.set()

View File

@@ -11,6 +11,7 @@ class MockHTTPStream:
status_code: int = 200, status_code: int = 200,
) -> None: ) -> None:
self.chunks = chunks self.chunks = chunks
self.headers = {"content-type": "audio/pcm"}
self.status_code = status_code self.status_code = status_code
async def __aenter__(self): async def __aenter__(self):