alex.a.alves/voiceagent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: formaliza formatos de áudio e alinhamento PCM

Browse Source
This commit is contained in:
alex alves
2026-11-03 16:43:22 -03:00
parent 4d7ff46a36
commit 76d15b8e52
23 changed files with 349 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/voice_assistant/application/events.py
View File

@@ -2,6 +2,8 @@
from dataclasses import dataclass
from voice_assistant.audio.models import AudioFormat
@dataclass(frozen=True, slots=True)
class TranscriptPartial:
@@ -36,6 +38,7 @@ class SpeechChunk:
session_id: str
turn_id: int
audio: bytes
audio_format: AudioFormat
@dataclass(frozen=True, slots=True)

10
src/voice_assistant/application/voice_session.py
View File

@@ -16,6 +16,7 @@ from voice_assistant.application.events import (
TurnCompleted,
VoiceEvent,
)
from voice_assistant.audio.models import AudioFormat, CANONICAL_AUDIO_FORMAT
from voice_assistant.stt.adapter import STTAdapter
from voice_assistant.voice_pipeline import (
PipelineAgentResponse,
@@ -95,6 +96,10 @@ class VoiceSession:
def closed(self) -> bool:
return self._pipeline.session.closed
@property
def output_audio_format(self) -> AudioFormat:
return self._pipeline.tts.output_format
def validate_audio_input(self, turn_id: int) -> None:
self._require_open()
if self._audio_input_closed:
@@ -165,6 +170,7 @@ class VoiceSession:
self, turn_id: int, sequence_number: int, audio: bytes,
) -> None:
self.validate_audio_input(turn_id)
CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
self._pipeline.session.accept_audio_chunk(
turn_id, sequence_number, len(audio),
)
@@ -258,7 +264,7 @@ class VoiceSession:
elif isinstance(event, PipelineAgentResponse):
await self._emit(AgentReply(self.session_id, turn_id, event.text))
elif isinstance(event, PipelineAudioChunk):
await self._emit(SpeechChunk(self.session_id, turn_id, event.audio))
await self._emit(SpeechChunk(self.session_id, turn_id, event.audio, self.output_audio_format))
else:
result = await self._pipeline.process_transcript(
session_id=self.session_id, turn_id=turn_id, text=text,
@@ -268,7 +274,7 @@ class VoiceSession:
self._pipeline.synthesize_response(result.response_text)
) as audio_stream:
async for audio in audio_stream:
await self._emit(SpeechChunk(self.session_id, turn_id, audio))
await self._emit(SpeechChunk(self.session_id, turn_id, audio, self.output_audio_format))
self._pipeline.on_tts_complete(turn_id)
async def _finalize_turn(self, turn_id: int, stt: STTAdapter) -> None:

67
src/voice_assistant/audio/models.py
View File

@@ -1,58 +1,36 @@
"""
Canonical audio models for the Voice Assistant.
"""PCM format contracts shared by application, transports, and providers.
Purpose:
Define the provider-independent audio format used internally by
the Voice Core.
Responsibilities:
- Describe an audio format in a typed structure.
- Define the canonical audio format used by the Voice Core.
- Provide a shared audio contract for transports and providers.
Non-responsibilities:
- Encoding or decoding audio.
- Resampling audio.
- Converting provider-specific audio formats.
- Managing WebSocket audio frames.
- Performing STT or TTS operations.
Architecture:
Client / Transport
|
v
Canonical Audio
PCM S16LE / 16 kHz / mono
|
v
Voice Core
|
+-----+-----+
| |
v v
STT TTS
Adapter Adapter
Provider-specific format conversion, when required, belongs
inside the corresponding provider adapter.
The canonical input is mono PCM S16LE at 16 kHz. Output uses the format
explicitly declared by the TTS provider. Declaring a sample rate does not
convert or resample audio; provider adapters own any required conversion.
"""
from typing import Literal
from pydantic import BaseModel
from pydantic import BaseModel, ConfigDict, Field
class AudioFormat(BaseModel):
"""
Describe one PCM audio format.
"""An immutable PCM format; a frame contains one sample per channel."""
This model is independent from WebSocket, STT, TTS, or any
concrete provider.
"""
model_config = ConfigDict(frozen=True)
encoding: Literal["pcm_s16le"]
sample_rate: int
channels: int
sample_rate: int = Field(gt=0, strict=True)
channels: int = Field(gt=0, strict=True)
@property
def frame_size_bytes(self) -> int:
return 2 * self.channels
def validate_chunk(self, audio: bytes) -> None:
if not audio:
raise ValueError("Audio chunk cannot be empty.")
if len(audio) % self.frame_size_bytes:
raise ValueError(
"PCM chunk must contain complete frames of "
f"{self.frame_size_bytes} bytes."
)
CANONICAL_AUDIO_FORMAT = AudioFormat(
@@ -60,4 +38,3 @@ CANONICAL_AUDIO_FORMAT = AudioFormat(
sample_rate=16000,
channels=1,
)

1
src/voice_assistant/audio_gateway/app/main.py
View File

@@ -226,6 +226,7 @@ async def audio_socket(websocket: WebSocket) -> None:
await send_json(websocket, SessionReady(
protocol_version=PROTOCOL_VERSION, session_id=session.session_id,
audio=CANONICAL_AUDIO_FORMAT,
output_audio=session.output_audio_format,
))
continue
if session is None:

7
src/voice_assistant/audio_gateway/app/protocol.py
View File

@@ -10,9 +10,9 @@ from voice_assistant.audio.models import (
)
PROTOCOL_VERSION = "1.0"
AUDIO_ENCODING = "pcm_s16le"
AUDIO_SAMPLE_RATE = 16000
AUDIO_CHANNELS = 1
AUDIO_ENCODING = CANONICAL_AUDIO_FORMAT.encoding
AUDIO_SAMPLE_RATE = CANONICAL_AUDIO_FORMAT.sample_rate
AUDIO_CHANNELS = CANONICAL_AUDIO_FORMAT.channels
class SessionStart(BaseModel):
@@ -81,6 +81,7 @@ class SessionReady(BaseModel):
protocol_version: str
session_id: str
audio: AudioFormat
output_audio: AudioFormat
class TurnStarted(BaseModel):

16
src/voice_assistant/audio_gateway/web/pcm-player.js
View File

@@ -62,6 +62,22 @@ export class PCMPlayer {
return this.analyser;
}
configure(format) {
if (
format?.encoding !== "pcm_s16le" ||
format.channels !== 1 ||
!Number.isInteger(format.sample_rate) ||
format.sample_rate < 8000 ||
format.sample_rate > 96000
) {
throw new Error("Formato de áudio de saída incompatível.");
}
if (this.sources.size > 0) {
throw new Error("Não é possível trocar o formato durante a reprodução.");
}
this.sampleRate = format.sample_rate;
}
decodePCM(buffer) {
if (!(buffer instanceof ArrayBuffer)) {
throw new TypeError("PCM chunk must be an ArrayBuffer.");

2
src/voice_assistant/audio_gateway/web/voice-session.js
View File

@@ -102,7 +102,6 @@ export class VoiceSessionController {
this.player = new PCMPlayer({
audioContext,
sampleRate: 48000,
onPlaybackStart: () => {
if (
@@ -547,6 +546,7 @@ export class VoiceSessionController {
handleGatewayEvent(event) {
switch (event.type) {
case "session.ready":
this.player.configure(event.output_audio);
this.startListening();
break;

7
src/voice_assistant/stt/mock.py
View File

@@ -34,6 +34,8 @@ Architecture:
import asyncio
from collections.abc import AsyncIterator
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
from .adapter import STTAdapter
from .models import TranscriptEvent
@@ -56,10 +58,7 @@ class MockSTTAdapter(STTAdapter):
"STT session has not been started."
)
if not audio:
raise ValueError(
"Audio chunk cannot be empty."
)
CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
async def stop_session(self) -> None:
if not self._started:

3
src/voice_assistant/stt/nvidia.py
View File

@@ -105,8 +105,7 @@ class NvidiaSTTAdapter(STTAdapter):
if not self._started or self._websocket is None:
raise RuntimeError("STT session has not been started.")
if not audio:
raise ValueError("Audio chunk cannot be empty.")
CANONICAL_AUDIO_FORMAT.validate_chunk(audio)
await self._send_json(
{

11
src/voice_assistant/tts/adapter.py
View File

@@ -3,13 +3,22 @@
from abc import ABC, abstractmethod
from collections.abc import AsyncGenerator
from voice_assistant.audio.models import AudioFormat
class TTSAdapter(ABC):
@property
@abstractmethod
def output_format(self) -> AudioFormat:
"""Declare the actual PCM format emitted throughout the session."""
raise NotImplementedError
@abstractmethod
def synthesize(self, text: str) -> AsyncGenerator[bytes, None]:
"""Return an audio generator consumed directly with async for.
The caller closes it on early exit. Implementations must release
Each yielded chunk must contain nonempty, complete PCM frames
matching output_format. The caller closes it on early exit. Implementations must release
stream-owned resources in finally blocks and propagate cancellation
and provider errors. The caller does not await synthesize itself.
"""

6
src/voice_assistant/tts/mock.py
View File

@@ -1,9 +1,15 @@
from collections.abc import AsyncGenerator
from voice_assistant.audio.models import AudioFormat
from .adapter import TTSAdapter
class MockTTSAdapter(TTSAdapter):
output_format = AudioFormat(
encoding="pcm_s16le", sample_rate=48000, channels=1,
)
def __init__(self) -> None:
self.calls: list[str] = []

28
src/voice_assistant/tts/voxcpm.py
View File

@@ -2,6 +2,8 @@ from collections.abc import AsyncGenerator
import httpx
from voice_assistant.audio.models import AudioFormat
from .adapter import TTSAdapter
@@ -15,7 +17,15 @@ class VoxCPMTTSAdapter(TTSAdapter):
stream: bool = True,
stream_format: str = "audio",
timeout: float = 120.0,
sample_rate: int = 48000,
) -> None:
if response_format != "pcm":
raise ValueError("VoxCPM must emit raw PCM audio.")
if stream_format != "audio":
raise ValueError("VoxCPM must emit audio bytes, not SSE events.")
self._output_format = AudioFormat(
encoding="pcm_s16le", sample_rate=sample_rate, channels=1,
)
self.server = server.rstrip("/")
self.model = model
self.voice = voice
@@ -24,6 +34,10 @@ class VoxCPMTTSAdapter(TTSAdapter):
self.stream_format = stream_format
self.timeout = timeout
@property
def output_format(self) -> AudioFormat:
return self._output_format
async def synthesize(
self,
text: str,
@@ -51,7 +65,17 @@ class VoxCPMTTSAdapter(TTSAdapter):
json=payload,
) as response:
response.raise_for_status()
content_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if content_type not in {"audio/pcm", "application/octet-stream"}:
raise ValueError("TTS response must contain raw PCM audio.")
remainder = b""
frame_size = self.output_format.frame_size_bytes
async for chunk in response.aiter_bytes():
if chunk:
yield chunk
data = remainder + chunk
aligned_size = len(data) - len(data) % frame_size
remainder = data[aligned_size:]
if aligned_size:
yield data[:aligned_size]
if remainder:
raise ValueError("TTS stream ended with an incomplete PCM frame.")

2
src/voice_assistant/voice_pipeline.py
View File

@@ -183,6 +183,7 @@ class VoicePipeline:
"""
async with aclosing(self.tts.synthesize(response_text)) as audio_stream:
async for audio_chunk in audio_stream:
self.tts.output_format.validate_chunk(audio_chunk)
yield audio_chunk
@@ -280,6 +281,7 @@ class VoicePipeline:
self.tts.synthesize(segment)
) as audio_stream:
async for audio_chunk in audio_stream:
self.tts.output_format.validate_chunk(audio_chunk)
await events.put(
PipelineAudioChunk(audio=audio_chunk)
)