refactor: formaliza formatos de áudio e alinhamento PCM
This commit is contained in:
@@ -311,6 +311,7 @@ The `staging` and `production` files are present, but deployment readiness for t
|
|||||||
| Endpoint | `ws://127.0.0.1:8001/ws/audio` |
|
| Endpoint | `ws://127.0.0.1:8001/ws/audio` |
|
||||||
| Protocol version | `1.0` |
|
| Protocol version | `1.0` |
|
||||||
| Input audio | PCM signed 16-bit little-endian, 16 kHz, mono |
|
| Input audio | PCM signed 16-bit little-endian, 16 kHz, mono |
|
||||||
|
| Output audio | Declared by `session.ready.output_audio`; current VoxCPM: PCM S16LE, 48 kHz, mono |
|
||||||
| Transport: | JSON control messages and binary PCM frames. |
|
| Transport: | JSON control messages and binary PCM frames. |
|
||||||
|
|
||||||
### Typical exchange
|
### Typical exchange
|
||||||
@@ -340,7 +341,7 @@ sequenceDiagram
|
|||||||
G-->>C: turn.ended
|
G-->>C: turn.ended
|
||||||
```
|
```
|
||||||
|
|
||||||
STT events may arrive while input audio is still streaming. Binary audio input is preceded by an `audio.chunk` control message containing `turn_id` and `sequence_number`. The Gateway limits each binary chunk to 64 KiB.
|
STT events may arrive while input audio is still streaming. Binary audio input is preceded by an `audio.chunk` control message containing `turn_id` and `sequence_number`. The Gateway limits each binary chunk to 64 KiB. Input frames must contain complete PCM samples; rejected frames do not advance the audio sequence. `session.ready.audio` describes input and `session.ready.output_audio` describes output. The browser configures playback from the declared output format. VoxCPM reassembles HTTP fragments into complete PCM frames and rejects a truncated final sample. Its `sample_rate` option declares the expected native output rate; it does not request resampling from the external service.
|
||||||
|
|
||||||
### Interruption and session closure
|
### Interruption and session closure
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ tts:
|
|||||||
provider: voxcpm
|
provider: voxcpm
|
||||||
options:
|
options:
|
||||||
server: "http://127.0.0.1:8000"
|
server: "http://127.0.0.1:8000"
|
||||||
|
sample_rate: 48000
|
||||||
model: "openbmb/VoxCPM2"
|
model: "openbmb/VoxCPM2"
|
||||||
voice: "default"
|
voice: "default"
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from voice_assistant.audio.models import AudioFormat
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class TranscriptPartial:
|
class TranscriptPartial:
|
||||||
@@ -36,6 +38,7 @@ class SpeechChunk:
|
|||||||
session_id: str
|
session_id: str
|
||||||
turn_id: int
|
turn_id: int
|
||||||
audio: bytes
|
audio: bytes
|
||||||
|
audio_format: AudioFormat
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from voice_assistant.application.events import (
|
|||||||
TurnCompleted,
|
TurnCompleted,
|
||||||
VoiceEvent,
|
VoiceEvent,
|
||||||
)
|
)
|
||||||
|
from voice_assistant.audio.models import AudioFormat, CANONICAL_AUDIO_FORMAT
|
||||||
from voice_assistant.stt.adapter import STTAdapter
|
from voice_assistant.stt.adapter import STTAdapter
|
||||||
from voice_assistant.voice_pipeline import (
|
from voice_assistant.voice_pipeline import (
|
||||||
PipelineAgentResponse,
|
PipelineAgentResponse,
|
||||||
@@ -95,6 +96,10 @@ class VoiceSession:
|
|||||||
def closed(self) -> bool:
|
def closed(self) -> bool:
|
||||||
return self._pipeline.session.closed
|
return self._pipeline.session.closed
|
||||||
|
|
||||||
|
@property
|
||||||
|
def output_audio_format(self) -> AudioFormat:
|
||||||
|
return self._pipeline.tts.output_format
|
||||||
|
|
||||||
def validate_audio_input(self, turn_id: int) -> None:
|
def validate_audio_input(self, turn_id: int) -> None:
|
||||||
self._require_open()
|
self._require_open()
|
||||||
if self._audio_input_closed:
|
if self._audio_input_closed:
|
||||||
@@ -165,6 +170,7 @@ class VoiceSession:
|
|||||||
self, turn_id: int, sequence_number: int, audio: bytes,
|
self, turn_id: int, sequence_number: int, audio: bytes,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.validate_audio_input(turn_id)
|
self.validate_audio_input(turn_id)
|
||||||
|
CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
|
||||||
self._pipeline.session.accept_audio_chunk(
|
self._pipeline.session.accept_audio_chunk(
|
||||||
turn_id, sequence_number, len(audio),
|
turn_id, sequence_number, len(audio),
|
||||||
)
|
)
|
||||||
@@ -258,7 +264,7 @@ class VoiceSession:
|
|||||||
elif isinstance(event, PipelineAgentResponse):
|
elif isinstance(event, PipelineAgentResponse):
|
||||||
await self._emit(AgentReply(self.session_id, turn_id, event.text))
|
await self._emit(AgentReply(self.session_id, turn_id, event.text))
|
||||||
elif isinstance(event, PipelineAudioChunk):
|
elif isinstance(event, PipelineAudioChunk):
|
||||||
await self._emit(SpeechChunk(self.session_id, turn_id, event.audio))
|
await self._emit(SpeechChunk(self.session_id, turn_id, event.audio, self.output_audio_format))
|
||||||
else:
|
else:
|
||||||
result = await self._pipeline.process_transcript(
|
result = await self._pipeline.process_transcript(
|
||||||
session_id=self.session_id, turn_id=turn_id, text=text,
|
session_id=self.session_id, turn_id=turn_id, text=text,
|
||||||
@@ -268,7 +274,7 @@ class VoiceSession:
|
|||||||
self._pipeline.synthesize_response(result.response_text)
|
self._pipeline.synthesize_response(result.response_text)
|
||||||
) as audio_stream:
|
) as audio_stream:
|
||||||
async for audio in audio_stream:
|
async for audio in audio_stream:
|
||||||
await self._emit(SpeechChunk(self.session_id, turn_id, audio))
|
await self._emit(SpeechChunk(self.session_id, turn_id, audio, self.output_audio_format))
|
||||||
self._pipeline.on_tts_complete(turn_id)
|
self._pipeline.on_tts_complete(turn_id)
|
||||||
|
|
||||||
async def _finalize_turn(self, turn_id: int, stt: STTAdapter) -> None:
|
async def _finalize_turn(self, turn_id: int, stt: STTAdapter) -> None:
|
||||||
|
|||||||
@@ -1,58 +1,36 @@
|
|||||||
"""
|
"""PCM format contracts shared by application, transports, and providers.
|
||||||
Canonical audio models for the Voice Assistant.
|
|
||||||
|
|
||||||
Purpose:
|
The canonical input is mono PCM S16LE at 16 kHz. Output uses the format
|
||||||
Define the provider-independent audio format used internally by
|
explicitly declared by the TTS provider. Declaring a sample rate does not
|
||||||
the Voice Core.
|
convert or resample audio; provider adapters own any required conversion.
|
||||||
|
|
||||||
Responsibilities:
|
|
||||||
- Describe an audio format in a typed structure.
|
|
||||||
- Define the canonical audio format used by the Voice Core.
|
|
||||||
- Provide a shared audio contract for transports and providers.
|
|
||||||
|
|
||||||
Non-responsibilities:
|
|
||||||
- Encoding or decoding audio.
|
|
||||||
- Resampling audio.
|
|
||||||
- Converting provider-specific audio formats.
|
|
||||||
- Managing WebSocket audio frames.
|
|
||||||
- Performing STT or TTS operations.
|
|
||||||
|
|
||||||
Architecture:
|
|
||||||
Client / Transport
|
|
||||||
|
|
|
||||||
v
|
|
||||||
Canonical Audio
|
|
||||||
PCM S16LE / 16 kHz / mono
|
|
||||||
|
|
|
||||||
v
|
|
||||||
Voice Core
|
|
||||||
|
|
|
||||||
+-----+-----+
|
|
||||||
| |
|
|
||||||
v v
|
|
||||||
STT TTS
|
|
||||||
Adapter Adapter
|
|
||||||
|
|
||||||
Provider-specific format conversion, when required, belongs
|
|
||||||
inside the corresponding provider adapter.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
|
|
||||||
class AudioFormat(BaseModel):
|
class AudioFormat(BaseModel):
|
||||||
"""
|
"""An immutable PCM format; a frame contains one sample per channel."""
|
||||||
Describe one PCM audio format.
|
|
||||||
|
|
||||||
This model is independent from WebSocket, STT, TTS, or any
|
model_config = ConfigDict(frozen=True)
|
||||||
concrete provider.
|
|
||||||
"""
|
|
||||||
|
|
||||||
encoding: Literal["pcm_s16le"]
|
encoding: Literal["pcm_s16le"]
|
||||||
sample_rate: int
|
sample_rate: int = Field(gt=0, strict=True)
|
||||||
channels: int
|
channels: int = Field(gt=0, strict=True)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def frame_size_bytes(self) -> int:
|
||||||
|
return 2 * self.channels
|
||||||
|
|
||||||
|
def validate_chunk(self, audio: bytes) -> None:
|
||||||
|
if not audio:
|
||||||
|
raise ValueError("Audio chunk cannot be empty.")
|
||||||
|
if len(audio) % self.frame_size_bytes:
|
||||||
|
raise ValueError(
|
||||||
|
"PCM chunk must contain complete frames of "
|
||||||
|
f"{self.frame_size_bytes} bytes."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
CANONICAL_AUDIO_FORMAT = AudioFormat(
|
CANONICAL_AUDIO_FORMAT = AudioFormat(
|
||||||
@@ -60,4 +38,3 @@ CANONICAL_AUDIO_FORMAT = AudioFormat(
|
|||||||
sample_rate=16000,
|
sample_rate=16000,
|
||||||
channels=1,
|
channels=1,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -226,6 +226,7 @@ async def audio_socket(websocket: WebSocket) -> None:
|
|||||||
await send_json(websocket, SessionReady(
|
await send_json(websocket, SessionReady(
|
||||||
protocol_version=PROTOCOL_VERSION, session_id=session.session_id,
|
protocol_version=PROTOCOL_VERSION, session_id=session.session_id,
|
||||||
audio=CANONICAL_AUDIO_FORMAT,
|
audio=CANONICAL_AUDIO_FORMAT,
|
||||||
|
output_audio=session.output_audio_format,
|
||||||
))
|
))
|
||||||
continue
|
continue
|
||||||
if session is None:
|
if session is None:
|
||||||
|
|||||||
@@ -10,9 +10,9 @@ from voice_assistant.audio.models import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
PROTOCOL_VERSION = "1.0"
|
PROTOCOL_VERSION = "1.0"
|
||||||
AUDIO_ENCODING = "pcm_s16le"
|
AUDIO_ENCODING = CANONICAL_AUDIO_FORMAT.encoding
|
||||||
AUDIO_SAMPLE_RATE = 16000
|
AUDIO_SAMPLE_RATE = CANONICAL_AUDIO_FORMAT.sample_rate
|
||||||
AUDIO_CHANNELS = 1
|
AUDIO_CHANNELS = CANONICAL_AUDIO_FORMAT.channels
|
||||||
|
|
||||||
|
|
||||||
class SessionStart(BaseModel):
|
class SessionStart(BaseModel):
|
||||||
@@ -81,6 +81,7 @@ class SessionReady(BaseModel):
|
|||||||
protocol_version: str
|
protocol_version: str
|
||||||
session_id: str
|
session_id: str
|
||||||
audio: AudioFormat
|
audio: AudioFormat
|
||||||
|
output_audio: AudioFormat
|
||||||
|
|
||||||
|
|
||||||
class TurnStarted(BaseModel):
|
class TurnStarted(BaseModel):
|
||||||
|
|||||||
@@ -62,6 +62,22 @@ export class PCMPlayer {
|
|||||||
return this.analyser;
|
return this.analyser;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
configure(format) {
|
||||||
|
if (
|
||||||
|
format?.encoding !== "pcm_s16le" ||
|
||||||
|
format.channels !== 1 ||
|
||||||
|
!Number.isInteger(format.sample_rate) ||
|
||||||
|
format.sample_rate < 8000 ||
|
||||||
|
format.sample_rate > 96000
|
||||||
|
) {
|
||||||
|
throw new Error("Formato de áudio de saída incompatível.");
|
||||||
|
}
|
||||||
|
if (this.sources.size > 0) {
|
||||||
|
throw new Error("Não é possível trocar o formato durante a reprodução.");
|
||||||
|
}
|
||||||
|
this.sampleRate = format.sample_rate;
|
||||||
|
}
|
||||||
|
|
||||||
decodePCM(buffer) {
|
decodePCM(buffer) {
|
||||||
if (!(buffer instanceof ArrayBuffer)) {
|
if (!(buffer instanceof ArrayBuffer)) {
|
||||||
throw new TypeError("PCM chunk must be an ArrayBuffer.");
|
throw new TypeError("PCM chunk must be an ArrayBuffer.");
|
||||||
|
|||||||
@@ -102,7 +102,6 @@ export class VoiceSessionController {
|
|||||||
|
|
||||||
this.player = new PCMPlayer({
|
this.player = new PCMPlayer({
|
||||||
audioContext,
|
audioContext,
|
||||||
sampleRate: 48000,
|
|
||||||
|
|
||||||
onPlaybackStart: () => {
|
onPlaybackStart: () => {
|
||||||
if (
|
if (
|
||||||
@@ -547,6 +546,7 @@ export class VoiceSessionController {
|
|||||||
handleGatewayEvent(event) {
|
handleGatewayEvent(event) {
|
||||||
switch (event.type) {
|
switch (event.type) {
|
||||||
case "session.ready":
|
case "session.ready":
|
||||||
|
this.player.configure(event.output_audio);
|
||||||
this.startListening();
|
this.startListening();
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
|||||||
@@ -34,6 +34,8 @@ Architecture:
|
|||||||
import asyncio
|
import asyncio
|
||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
|
|
||||||
|
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
|
||||||
|
|
||||||
from .adapter import STTAdapter
|
from .adapter import STTAdapter
|
||||||
from .models import TranscriptEvent
|
from .models import TranscriptEvent
|
||||||
|
|
||||||
@@ -56,10 +58,7 @@ class MockSTTAdapter(STTAdapter):
|
|||||||
"STT session has not been started."
|
"STT session has not been started."
|
||||||
)
|
)
|
||||||
|
|
||||||
if not audio:
|
CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
|
||||||
raise ValueError(
|
|
||||||
"Audio chunk cannot be empty."
|
|
||||||
)
|
|
||||||
|
|
||||||
async def stop_session(self) -> None:
|
async def stop_session(self) -> None:
|
||||||
if not self._started:
|
if not self._started:
|
||||||
|
|||||||
@@ -105,8 +105,7 @@ class NvidiaSTTAdapter(STTAdapter):
|
|||||||
if not self._started or self._websocket is None:
|
if not self._started or self._websocket is None:
|
||||||
raise RuntimeError("STT session has not been started.")
|
raise RuntimeError("STT session has not been started.")
|
||||||
|
|
||||||
if not audio:
|
CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
|
||||||
raise ValueError("Audio chunk cannot be empty.")
|
|
||||||
|
|
||||||
await self._send_json(
|
await self._send_json(
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -3,13 +3,22 @@
|
|||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
|
|
||||||
|
from voice_assistant.audio.models import AudioFormat
|
||||||
|
|
||||||
|
|
||||||
class TTSAdapter(ABC):
|
class TTSAdapter(ABC):
|
||||||
|
@property
|
||||||
|
@abstractmethod
|
||||||
|
def output_format(self) -> AudioFormat:
|
||||||
|
"""Declare the actual PCM format emitted throughout the session."""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def synthesize(self, text: str) -> AsyncGenerator[bytes, None]:
|
def synthesize(self, text: str) -> AsyncGenerator[bytes, None]:
|
||||||
"""Return an audio generator consumed directly with async for.
|
"""Return an audio generator consumed directly with async for.
|
||||||
|
|
||||||
The caller closes it on early exit. Implementations must release
|
Each yielded chunk must contain nonempty, complete PCM frames
|
||||||
|
matching output_format. The caller closes it on early exit. Implementations must release
|
||||||
stream-owned resources in finally blocks and propagate cancellation
|
stream-owned resources in finally blocks and propagate cancellation
|
||||||
and provider errors. The caller does not await synthesize itself.
|
and provider errors. The caller does not await synthesize itself.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1,9 +1,15 @@
|
|||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
|
|
||||||
|
from voice_assistant.audio.models import AudioFormat
|
||||||
|
|
||||||
from .adapter import TTSAdapter
|
from .adapter import TTSAdapter
|
||||||
|
|
||||||
|
|
||||||
class MockTTSAdapter(TTSAdapter):
|
class MockTTSAdapter(TTSAdapter):
|
||||||
|
output_format = AudioFormat(
|
||||||
|
encoding="pcm_s16le", sample_rate=48000, channels=1,
|
||||||
|
)
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.calls: list[str] = []
|
self.calls: list[str] = []
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ from collections.abc import AsyncGenerator
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
|
from voice_assistant.audio.models import AudioFormat
|
||||||
|
|
||||||
from .adapter import TTSAdapter
|
from .adapter import TTSAdapter
|
||||||
|
|
||||||
|
|
||||||
@@ -15,7 +17,15 @@ class VoxCPMTTSAdapter(TTSAdapter):
|
|||||||
stream: bool = True,
|
stream: bool = True,
|
||||||
stream_format: str = "audio",
|
stream_format: str = "audio",
|
||||||
timeout: float = 120.0,
|
timeout: float = 120.0,
|
||||||
|
sample_rate: int = 48000,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
if response_format != "pcm":
|
||||||
|
raise ValueError("VoxCPM must emit raw PCM audio.")
|
||||||
|
if stream_format != "audio":
|
||||||
|
raise ValueError("VoxCPM must emit audio bytes, not SSE events.")
|
||||||
|
self._output_format = AudioFormat(
|
||||||
|
encoding="pcm_s16le", sample_rate=sample_rate, channels=1,
|
||||||
|
)
|
||||||
self.server = server.rstrip("/")
|
self.server = server.rstrip("/")
|
||||||
self.model = model
|
self.model = model
|
||||||
self.voice = voice
|
self.voice = voice
|
||||||
@@ -24,6 +34,10 @@ class VoxCPMTTSAdapter(TTSAdapter):
|
|||||||
self.stream_format = stream_format
|
self.stream_format = stream_format
|
||||||
self.timeout = timeout
|
self.timeout = timeout
|
||||||
|
|
||||||
|
@property
|
||||||
|
def output_format(self) -> AudioFormat:
|
||||||
|
return self._output_format
|
||||||
|
|
||||||
async def synthesize(
|
async def synthesize(
|
||||||
self,
|
self,
|
||||||
text: str,
|
text: str,
|
||||||
@@ -51,7 +65,17 @@ class VoxCPMTTSAdapter(TTSAdapter):
|
|||||||
json=payload,
|
json=payload,
|
||||||
) as response:
|
) as response:
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
content_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||||
|
if content_type not in {"audio/pcm", "application/octet-stream"}:
|
||||||
|
raise ValueError("TTS response must contain raw PCM audio.")
|
||||||
|
|
||||||
|
remainder = b""
|
||||||
|
frame_size = self.output_format.frame_size_bytes
|
||||||
async for chunk in response.aiter_bytes():
|
async for chunk in response.aiter_bytes():
|
||||||
if chunk:
|
data = remainder + chunk
|
||||||
yield chunk
|
aligned_size = len(data) - len(data) % frame_size
|
||||||
|
remainder = data[aligned_size:]
|
||||||
|
if aligned_size:
|
||||||
|
yield data[:aligned_size]
|
||||||
|
if remainder:
|
||||||
|
raise ValueError("TTS stream ended with an incomplete PCM frame.")
|
||||||
@@ -183,6 +183,7 @@ class VoicePipeline:
|
|||||||
"""
|
"""
|
||||||
async with aclosing(self.tts.synthesize(response_text)) as audio_stream:
|
async with aclosing(self.tts.synthesize(response_text)) as audio_stream:
|
||||||
async for audio_chunk in audio_stream:
|
async for audio_chunk in audio_stream:
|
||||||
|
self.tts.output_format.validate_chunk(audio_chunk)
|
||||||
yield audio_chunk
|
yield audio_chunk
|
||||||
|
|
||||||
|
|
||||||
@@ -280,6 +281,7 @@ class VoicePipeline:
|
|||||||
self.tts.synthesize(segment)
|
self.tts.synthesize(segment)
|
||||||
) as audio_stream:
|
) as audio_stream:
|
||||||
async for audio_chunk in audio_stream:
|
async for audio_chunk in audio_stream:
|
||||||
|
self.tts.output_format.validate_chunk(audio_chunk)
|
||||||
await events.put(
|
await events.put(
|
||||||
PipelineAudioChunk(audio=audio_chunk)
|
PipelineAudioChunk(audio=audio_chunk)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -223,3 +223,35 @@ test("late PCM after stop is ignored", async () => {
|
|||||||
assert.equal(player.playing, false);
|
assert.equal(player.playing, false);
|
||||||
assert.equal(player.sources.size, 0);
|
assert.equal(player.sources.size, 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
test("uses the output sample rate declared by the server", async () => {
|
||||||
|
const context = new FakeAudioContext();
|
||||||
|
const player = new PCMPlayer({ audioContext: context });
|
||||||
|
player.configure({ encoding: "pcm_s16le", sample_rate: 24000, channels: 1 });
|
||||||
|
await player.enqueue(pcm([100, 200, 300, 400]));
|
||||||
|
assert.equal(player.sampleRate, 24000);
|
||||||
|
assert.equal(context.sources[0].buffer.duration, 4 / 24000);
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const format of [
|
||||||
|
undefined,
|
||||||
|
{ encoding: "wav", sample_rate: 48000, channels: 1 },
|
||||||
|
{ encoding: "pcm_s16le", sample_rate: 48000, channels: 2 },
|
||||||
|
{ encoding: "pcm_s16le", sample_rate: "48000", channels: 1 },
|
||||||
|
{ encoding: "pcm_s16le", sample_rate: 0, channels: 1 },
|
||||||
|
{ encoding: "pcm_s16le", sample_rate: 192000, channels: 1 },
|
||||||
|
]) {
|
||||||
|
test(`rejects unsupported playback metadata: ${JSON.stringify(format)}`, () => {
|
||||||
|
const player = new PCMPlayer({ audioContext: new FakeAudioContext() });
|
||||||
|
assert.throws(() => player.configure(format), /Formato de áudio/);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
test("rejects format changes while audio is scheduled", async () => {
|
||||||
|
const player = new PCMPlayer({ audioContext: new FakeAudioContext() });
|
||||||
|
await player.enqueue(pcm([100, 200]));
|
||||||
|
assert.throws(() => player.configure({
|
||||||
|
encoding: "pcm_s16le", sample_rate: 24000, channels: 1,
|
||||||
|
}), /durante a reprodução/);
|
||||||
|
});
|
||||||
|
|||||||
@@ -733,3 +733,14 @@ test("barge-in resets player before new TTS response", () => {
|
|||||||
assert.equal(finished, false);
|
assert.equal(finished, false);
|
||||||
assert.equal(resetCount, 1);
|
assert.equal(resetCount, 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
test("session.ready configures output before starting microphone capture", () => {
|
||||||
|
const { controller } = makeController();
|
||||||
|
const operations = [];
|
||||||
|
const format = { encoding: "pcm_s16le", sample_rate: 24000, channels: 1 };
|
||||||
|
controller.player = { configure: (received) => operations.push(received) };
|
||||||
|
controller.startListening = () => operations.push("listen");
|
||||||
|
controller.handleGatewayEvent({ type: "session.ready", output_audio: format });
|
||||||
|
assert.deepEqual(operations, [format, "listen"]);
|
||||||
|
});
|
||||||
|
|||||||
175
tests/test_audio_contract.py
Normal file
175
tests/test_audio_contract.py
Normal file
@@ -0,0 +1,175 @@
|
|||||||
|
"""PCM framing, per-direction metadata, and HTTP fragmentation guarantees."""
|
||||||
|
|
||||||
|
from contextlib import aclosing
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
from pydantic import ValidationError
|
||||||
|
|
||||||
|
from voice_assistant.agent.mock import MockAgentAdapter
|
||||||
|
from voice_assistant.application.voice_session import VoiceSession
|
||||||
|
from voice_assistant.audio.models import AudioFormat, CANONICAL_AUDIO_FORMAT
|
||||||
|
from voice_assistant.audio_gateway.app import main
|
||||||
|
from voice_assistant.domain.session import SessionState
|
||||||
|
from voice_assistant.stt.mock import MockSTTAdapter
|
||||||
|
from voice_assistant.tts.mock import MockTTSAdapter
|
||||||
|
from voice_assistant.tts.voxcpm import VoxCPMTTSAdapter
|
||||||
|
from voice_assistant.turn_manager import TurnManager
|
||||||
|
from voice_assistant.voice_pipeline import VoicePipeline
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def anyio_backend():
|
||||||
|
return "asyncio"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("field", ["sample_rate", "channels"])
|
||||||
|
@pytest.mark.parametrize("value", [0, -1, True, "16000", 1.5])
|
||||||
|
def test_audio_format_requires_positive_integer_dimensions(field, value):
|
||||||
|
fields = {"encoding": "pcm_s16le", "sample_rate": 16000, "channels": 1}
|
||||||
|
fields[field] = value
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
AudioFormat(**fields)
|
||||||
|
|
||||||
|
|
||||||
|
def test_shared_format_cannot_be_mutated():
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
CANONICAL_AUDIO_FORMAT.sample_rate = 48000
|
||||||
|
assert CANONICAL_AUDIO_FORMAT.sample_rate == 16000
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("channels", [1, 2])
|
||||||
|
@pytest.mark.parametrize("size", [0, 1, 2, 3, 4, 5, 8])
|
||||||
|
def test_pcm_validation_requires_complete_channel_frames(channels, size):
|
||||||
|
format_ = AudioFormat(encoding="pcm_s16le", sample_rate=16000, channels=channels)
|
||||||
|
if not size or size % (2 * channels):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
format_.validate_chunk(bytes(size))
|
||||||
|
else:
|
||||||
|
format_.validate_chunk(bytes(size))
|
||||||
|
|
||||||
|
|
||||||
|
def make_pipeline(tts):
|
||||||
|
return VoicePipeline(
|
||||||
|
session=SessionState(), turn_manager=TurnManager(),
|
||||||
|
agent=MockAgentAdapter(), tts=tts,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_invalid_input_does_not_advance_sequence_or_reach_stt():
|
||||||
|
class STT(MockSTTAdapter):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.received = []
|
||||||
|
|
||||||
|
async def send_audio(self, audio):
|
||||||
|
await super().send_audio(audio)
|
||||||
|
self.received.append(audio)
|
||||||
|
|
||||||
|
async def emit(event):
|
||||||
|
pass
|
||||||
|
|
||||||
|
stt = STT()
|
||||||
|
session = VoiceSession(
|
||||||
|
pipeline=make_pipeline(MockTTSAdapter()), create_stt=lambda: stt, emit=emit,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
turn_id = await session.start_turn()
|
||||||
|
with pytest.raises(ValueError, match="complete frames"):
|
||||||
|
await session.send_audio(turn_id, 0, b"\x00")
|
||||||
|
assert session.expected_audio_sequence == 0
|
||||||
|
assert stt.received == []
|
||||||
|
await session.send_audio(turn_id, 0, b"\x00\x00")
|
||||||
|
assert session.expected_audio_sequence == 1
|
||||||
|
assert stt.received == [b"\x00\x00"]
|
||||||
|
finally:
|
||||||
|
await session.aclose()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_pipeline_rejects_misaligned_tts_output():
|
||||||
|
class TTS(MockTTSAdapter):
|
||||||
|
async def synthesize(self, text):
|
||||||
|
yield b"\x00"
|
||||||
|
|
||||||
|
async with aclosing(make_pipeline(TTS()).synthesize_response("Olá")) as stream:
|
||||||
|
with pytest.raises(ValueError, match="complete frames"):
|
||||||
|
await anext(stream)
|
||||||
|
|
||||||
|
|
||||||
|
class FragmentedStream(httpx.AsyncByteStream):
|
||||||
|
def __init__(self, chunks):
|
||||||
|
self.chunks = chunks
|
||||||
|
|
||||||
|
async def __aiter__(self):
|
||||||
|
for chunk in self.chunks:
|
||||||
|
yield chunk
|
||||||
|
|
||||||
|
|
||||||
|
def mock_tts_http(monkeypatch, chunks, content_type="audio/pcm"):
|
||||||
|
original_client = httpx.AsyncClient
|
||||||
|
|
||||||
|
def handle(request):
|
||||||
|
return httpx.Response(
|
||||||
|
200, headers={"content-type": content_type},
|
||||||
|
stream=FragmentedStream(chunks),
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
httpx, "AsyncClient",
|
||||||
|
lambda **kwargs: original_client(transport=httpx.MockTransport(handle), **kwargs),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
@pytest.mark.parametrize("chunks", [
|
||||||
|
[b"\x01", b"\x02\x03", b"", b"\x04"],
|
||||||
|
[b"\x01\x02\x03", b"\x04"],
|
||||||
|
[b"\x01\x02\x03\x04"],
|
||||||
|
])
|
||||||
|
async def test_voxcpm_reassembles_http_fragments_without_losing_bytes(monkeypatch, chunks):
|
||||||
|
mock_tts_http(monkeypatch, chunks)
|
||||||
|
tts = VoxCPMTTSAdapter(server="http://test", model="test", voice="default")
|
||||||
|
output = [chunk async for chunk in tts.synthesize("Olá")]
|
||||||
|
assert b"".join(output) == b"\x01\x02\x03\x04"
|
||||||
|
assert all(chunk and len(chunk) % 2 == 0 for chunk in output)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_voxcpm_rejects_truncated_final_frame(monkeypatch):
|
||||||
|
mock_tts_http(monkeypatch, [b"\x01\x02\x03"])
|
||||||
|
tts = VoxCPMTTSAdapter(server="http://test", model="test", voice="default")
|
||||||
|
with pytest.raises(ValueError, match="incomplete PCM frame"):
|
||||||
|
_ = [chunk async for chunk in tts.synthesize("Olá")]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
@pytest.mark.parametrize("content_type", ["audio/wav", "text/event-stream", "application/json"])
|
||||||
|
async def test_voxcpm_rejects_non_pcm_response(monkeypatch, content_type):
|
||||||
|
mock_tts_http(monkeypatch, [b"data"], content_type)
|
||||||
|
tts = VoxCPMTTSAdapter(server="http://test", model="test", voice="default")
|
||||||
|
with pytest.raises(ValueError, match="raw PCM"):
|
||||||
|
_ = [chunk async for chunk in tts.synthesize("Olá")]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("options", [{"response_format": "wav"}, {"stream_format": "sse"}])
|
||||||
|
def test_voxcpm_rejects_incompatible_transport_settings(options):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
VoxCPMTTSAdapter(server="http://test", model="test", voice="default", **options)
|
||||||
|
|
||||||
|
|
||||||
|
def test_gateway_announces_provider_output_format_and_preserves_input(monkeypatch):
|
||||||
|
class TTS(MockTTSAdapter):
|
||||||
|
output_format = AudioFormat(encoding="pcm_s16le", sample_rate=24000, channels=1)
|
||||||
|
|
||||||
|
monkeypatch.setattr(main, "create_tts_adapter", lambda **kwargs: TTS())
|
||||||
|
monkeypatch.setattr(main, "create_agent_adapter", lambda **kwargs: MockAgentAdapter())
|
||||||
|
with TestClient(main.app).websocket_connect("/ws/audio") as socket:
|
||||||
|
socket.send_json({"type": "session.start"})
|
||||||
|
ready = socket.receive_json()
|
||||||
|
assert ready["audio"]["sample_rate"] == 16000
|
||||||
|
assert ready["output_audio"] == {
|
||||||
|
"encoding": "pcm_s16le", "sample_rate": 24000, "channels": 1,
|
||||||
|
}
|
||||||
@@ -1,5 +1,4 @@
|
|||||||
from voice_assistant.agent.adapter import StreamingAgentAdapter
|
from voice_assistant.agent.adapter import StreamingAgentAdapter
|
||||||
from voice_assistant.tts.adapter import TTSAdapter
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
@@ -1865,9 +1864,9 @@ def test_websocket_streaming_agent_and_tts():
|
|||||||
yield "Olá, como posso ajudar?"
|
yield "Olá, como posso ajudar?"
|
||||||
yield " Estou à disposição."
|
yield " Estou à disposição."
|
||||||
|
|
||||||
class StreamingTestTTS(TTSAdapter):
|
class StreamingTestTTS(MockTTSAdapter):
|
||||||
async def synthesize(self, text):
|
async def synthesize(self, text):
|
||||||
yield text.encode("utf-8")
|
yield text.encode("utf-16-le")
|
||||||
|
|
||||||
with patch.object(
|
with patch.object(
|
||||||
main,
|
main,
|
||||||
@@ -1967,7 +1966,7 @@ def test_websocket_interrupt_during_streaming_tts():
|
|||||||
async def stream(self, session_id, turn_id, text):
|
async def stream(self, session_id, turn_id, text):
|
||||||
yield "Olá, esta é uma resposta longa."
|
yield "Olá, esta é uma resposta longa."
|
||||||
|
|
||||||
class SlowStreamingTTS(TTSAdapter):
|
class SlowStreamingTTS(MockTTSAdapter):
|
||||||
async def synthesize(self, text):
|
async def synthesize(self, text):
|
||||||
try:
|
try:
|
||||||
tts_started.set()
|
tts_started.set()
|
||||||
@@ -2106,7 +2105,7 @@ def test_websocket_session_end_during_streaming_tts():
|
|||||||
async def stream(self, session_id, turn_id, text):
|
async def stream(self, session_id, turn_id, text):
|
||||||
yield "Esta é uma resposta longa."
|
yield "Esta é uma resposta longa."
|
||||||
|
|
||||||
class SlowStreamingTTS(TTSAdapter):
|
class SlowStreamingTTS(MockTTSAdapter):
|
||||||
async def synthesize(self, text):
|
async def synthesize(self, text):
|
||||||
try:
|
try:
|
||||||
yield b"\x01\x02\x03\x04"
|
yield b"\x01\x02\x03\x04"
|
||||||
@@ -2303,7 +2302,7 @@ def test_websocket_interrupt_wins_pending_tts_completion():
|
|||||||
async def stream(self, session_id, turn_id, text):
|
async def stream(self, session_id, turn_id, text):
|
||||||
yield "Esta resposta será interrompida."
|
yield "Esta resposta será interrompida."
|
||||||
|
|
||||||
class ControlledTTS(TTSAdapter):
|
class ControlledTTS(MockTTSAdapter):
|
||||||
async def synthesize(self, text):
|
async def synthesize(self, text):
|
||||||
try:
|
try:
|
||||||
yield b"\x01\x02\x03\x04"
|
yield b"\x01\x02\x03\x04"
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ Non-responsibilities:
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
|
from voice_assistant.audio.models import AudioFormat, CANONICAL_AUDIO_FORMAT
|
||||||
from voice_assistant.audio_gateway.app.protocol import (
|
from voice_assistant.audio_gateway.app.protocol import (
|
||||||
AgentDelta,
|
AgentDelta,
|
||||||
AgentResponse,
|
AgentResponse,
|
||||||
@@ -83,6 +83,7 @@ def test_session_ready_output():
|
|||||||
protocol_version="1.0",
|
protocol_version="1.0",
|
||||||
session_id="session-1",
|
session_id="session-1",
|
||||||
audio=CANONICAL_AUDIO_FORMAT,
|
audio=CANONICAL_AUDIO_FORMAT,
|
||||||
|
output_audio=AudioFormat(encoding="pcm_s16le", sample_rate=48000, channels=1),
|
||||||
)
|
)
|
||||||
|
|
||||||
assert event.model_dump() == {
|
assert event.model_dump() == {
|
||||||
@@ -94,6 +95,11 @@ def test_session_ready_output():
|
|||||||
"sample_rate": 16000,
|
"sample_rate": 16000,
|
||||||
"channels": 1,
|
"channels": 1,
|
||||||
},
|
},
|
||||||
|
"output_audio": {
|
||||||
|
"encoding": "pcm_s16le",
|
||||||
|
"sample_rate": 48000,
|
||||||
|
"channels": 1,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ from voice_assistant.application.events import SpeechChunk
|
|||||||
from voice_assistant.application.voice_session import VoiceSession
|
from voice_assistant.application.voice_session import VoiceSession
|
||||||
from voice_assistant.domain.session import SessionState
|
from voice_assistant.domain.session import SessionState
|
||||||
from voice_assistant.stt.mock import MockSTTAdapter
|
from voice_assistant.stt.mock import MockSTTAdapter
|
||||||
from voice_assistant.tts.adapter import TTSAdapter
|
|
||||||
from voice_assistant.tts.mock import MockTTSAdapter
|
from voice_assistant.tts.mock import MockTTSAdapter
|
||||||
from voice_assistant.turn_manager import TurnManager
|
from voice_assistant.turn_manager import TurnManager
|
||||||
from voice_assistant.voice_pipeline import VoicePipeline
|
from voice_assistant.voice_pipeline import VoicePipeline
|
||||||
@@ -93,17 +92,17 @@ async def test_streaming_cancellation_releases_stream():
|
|||||||
async def test_stopping_synthesis_early_closes_provider_generator():
|
async def test_stopping_synthesis_early_closes_provider_generator():
|
||||||
closed = asyncio.Event()
|
closed = asyncio.Event()
|
||||||
|
|
||||||
class TTS(TTSAdapter):
|
class TTS(MockTTSAdapter):
|
||||||
async def synthesize(self, text):
|
async def synthesize(self, text):
|
||||||
try:
|
try:
|
||||||
yield b"audio"
|
yield b"audi"
|
||||||
await asyncio.Event().wait()
|
await asyncio.Event().wait()
|
||||||
finally:
|
finally:
|
||||||
closed.set()
|
closed.set()
|
||||||
|
|
||||||
pipeline = make_pipeline(MockAgentAdapter(), TTS())
|
pipeline = make_pipeline(MockAgentAdapter(), TTS())
|
||||||
async with aclosing(pipeline.synthesize_response("Olá")) as stream:
|
async with aclosing(pipeline.synthesize_response("Olá")) as stream:
|
||||||
assert await anext(stream) == b"audio"
|
assert await anext(stream) == b"audi"
|
||||||
assert closed.is_set()
|
assert closed.is_set()
|
||||||
|
|
||||||
|
|
||||||
@@ -113,10 +112,10 @@ async def test_interrupt_during_delivery_closes_nonstreaming_tts():
|
|||||||
synthesis_closed = asyncio.Event()
|
synthesis_closed = asyncio.Event()
|
||||||
stt = MockSTTAdapter()
|
stt = MockSTTAdapter()
|
||||||
|
|
||||||
class TTS(TTSAdapter):
|
class TTS(MockTTSAdapter):
|
||||||
async def synthesize(self, text):
|
async def synthesize(self, text):
|
||||||
try:
|
try:
|
||||||
yield b"audio"
|
yield b"audi"
|
||||||
await asyncio.Event().wait()
|
await asyncio.Event().wait()
|
||||||
finally:
|
finally:
|
||||||
synthesis_closed.set()
|
synthesis_closed.set()
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
from voice_assistant.agent.adapter import StreamingAgentAdapter
|
from voice_assistant.agent.adapter import StreamingAgentAdapter
|
||||||
from voice_assistant.tts.adapter import TTSAdapter
|
|
||||||
"""
|
"""
|
||||||
Tests for the Voice Pipeline lifecycle orchestration.
|
Tests for the Voice Pipeline lifecycle orchestration.
|
||||||
|
|
||||||
@@ -850,9 +849,9 @@ async def test_stream_turn_response_produces_audio_before_agent_finishes():
|
|||||||
yield " Estou à disposição."
|
yield " Estou à disposição."
|
||||||
self.finished = True
|
self.finished = True
|
||||||
|
|
||||||
class StreamingTTS(TTSAdapter):
|
class StreamingTTS(MockTTSAdapter):
|
||||||
async def synthesize(self, text):
|
async def synthesize(self, text):
|
||||||
yield text.encode("utf-8")
|
yield text.encode("utf-16-le")
|
||||||
|
|
||||||
agent = StreamingAgent()
|
agent = StreamingAgent()
|
||||||
|
|
||||||
@@ -938,9 +937,9 @@ async def test_stream_turn_response_propagates_agent_failure():
|
|||||||
yield "Olá, como posso ajudar?"
|
yield "Olá, como posso ajudar?"
|
||||||
raise RuntimeError("Simulated OCI streaming failure.")
|
raise RuntimeError("Simulated OCI streaming failure.")
|
||||||
|
|
||||||
class StreamingTTS(TTSAdapter):
|
class StreamingTTS(MockTTSAdapter):
|
||||||
async def synthesize(self, text):
|
async def synthesize(self, text):
|
||||||
yield text.encode("utf-8")
|
yield text.encode("utf-16-le")
|
||||||
|
|
||||||
pipeline = VoicePipeline(
|
pipeline = VoicePipeline(
|
||||||
session=SessionState(session_id="agent-failure-session"),
|
session=SessionState(session_id="agent-failure-session"),
|
||||||
@@ -984,7 +983,7 @@ async def test_stream_turn_response_cancels_agent_on_tts_failure():
|
|||||||
finally:
|
finally:
|
||||||
self.cancelled.set()
|
self.cancelled.set()
|
||||||
|
|
||||||
class FailingTTS(TTSAdapter):
|
class FailingTTS(MockTTSAdapter):
|
||||||
async def synthesize(self, text):
|
async def synthesize(self, text):
|
||||||
raise RuntimeError("Simulated TTS failure.")
|
raise RuntimeError("Simulated TTS failure.")
|
||||||
yield b"" # Keep this method an async generator.
|
yield b"" # Keep this method an async generator.
|
||||||
@@ -1036,7 +1035,7 @@ async def test_stream_turn_response_external_cancellation():
|
|||||||
finally:
|
finally:
|
||||||
self.closed.set()
|
self.closed.set()
|
||||||
|
|
||||||
class BlockingTTS(TTSAdapter):
|
class BlockingTTS(MockTTSAdapter):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.started = asyncio.Event()
|
self.started = asyncio.Event()
|
||||||
self.closed = asyncio.Event()
|
self.closed = asyncio.Event()
|
||||||
@@ -1045,7 +1044,7 @@ async def test_stream_turn_response_external_cancellation():
|
|||||||
try:
|
try:
|
||||||
self.started.set()
|
self.started.set()
|
||||||
await asyncio.Event().wait()
|
await asyncio.Event().wait()
|
||||||
yield b"audio"
|
yield b"audi"
|
||||||
finally:
|
finally:
|
||||||
self.closed.set()
|
self.closed.set()
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ class MockHTTPStream:
|
|||||||
status_code: int = 200,
|
status_code: int = 200,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.chunks = chunks
|
self.chunks = chunks
|
||||||
|
self.headers = {"content-type": "audio/pcm"}
|
||||||
self.status_code = status_code
|
self.status_code = status_code
|
||||||
|
|
||||||
async def __aenter__(self):
|
async def __aenter__(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user