alex.a.alves/voiceagent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: migra cliente web para áudio contínuo com VAD no backend

Browse Source
This commit is contained in:
alex alves
2026-11-03 18:34:45 -03:00
parent bdd2f6f67d
commit 7cbe86ed39
14 changed files with 834 additions and 1574 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/voice_assistant/application/voice_session.py
View File

@@ -127,6 +127,10 @@ class VoiceSession:
def output_audio_format(self) -> AudioFormat:
return self._pipeline.tts.output_format
@property
def max_input_chunk_bytes(self) -> int:
return self._max_input_chunk_bytes
def validate_audio_input(self, turn_id: int) -> None:
self._require_open()
if self._audio_input_closed:

82
src/voice_assistant/audio_gateway/app/main.py
View File

@@ -19,7 +19,7 @@ from voice_assistant.application.voice_session import VoiceSession, VoiceSession
from voice_assistant.audio.models import CANONICAL_AUDIO_FORMAT
from voice_assistant.audio_gateway.app.protocol import (
PROTOCOL_VERSION, AUDIO_ENCODING, AUDIO_SAMPLE_RATE, AUDIO_CHANNELS,
AudioChunk, AudioStop, Interrupt, SessionEnd, SessionStart,
AudioChunk, InputAudioChunk, InputAudioEnd, AudioStop, Interrupt, SessionEnd, SessionStart,
AgentDelta, AgentResponse, Interrupted, SessionEnded, SessionReady,
STTFinal, STTPartial, TurnEnded, TurnStarted, error_event,
)
@@ -139,7 +139,10 @@ async def create_voice_session(
async def audio_socket(websocket: WebSocket) -> None:
await websocket.accept()
session: VoiceSession | None = None
pending_audio_chunk: AudioChunk | None = None
pending_audio_chunk: AudioChunk | InputAudioChunk | None = None
server_vad = False
expected_input_sequence = 0
max_audio_chunk_bytes = MAX_AUDIO_CHUNK_BYTES
async def report(
code: str, message: str, *, turn_id: int | None = None,
@@ -169,22 +172,27 @@ async def audio_socket(websocket: WebSocket) -> None:
chunk = pending_audio_chunk
pending_audio_chunk = None
audio = message["bytes"]
if len(audio) > MAX_AUDIO_CHUNK_BYTES:
turn_id = chunk.turn_id if isinstance(chunk, AudioChunk) else session.active_turn_id
if len(audio) > max_audio_chunk_bytes:
await report(
"MESSAGE_TOO_LARGE",
f"Audio chunk exceeds maximum size of {MAX_AUDIO_CHUNK_BYTES} bytes.",
turn_id=chunk.turn_id,
details={"max_bytes": MAX_AUDIO_CHUNK_BYTES, "received_bytes": len(audio)},
f"Audio chunk exceeds maximum size of {max_audio_chunk_bytes} bytes.",
turn_id=turn_id,
details={"max_bytes": max_audio_chunk_bytes, "received_bytes": len(audio)},
)
continue
try:
await session.send_audio(chunk.turn_id, chunk.sequence_number, audio)
if isinstance(chunk, InputAudioChunk):
await session.receive_audio(audio)
expected_input_sequence += 1
else:
await session.send_audio(chunk.turn_id, chunk.sequence_number, audio)
except VoiceSessionError as exc:
await report(exc.code, str(exc), turn_id=exc.turn_id, retryable=exc.retryable)
except (RuntimeError, ValueError) as exc:
await report(
"AUDIO_SEQUENCE_ERROR", str(exc), turn_id=chunk.turn_id,
details={"expected_sequence_number": session.expected_audio_sequence},
"AUDIO_SEQUENCE_ERROR", str(exc), turn_id=turn_id,
details={"expected_sequence_number": expected_input_sequence if server_vad else session.expected_audio_sequence},
)
continue
@@ -232,11 +240,28 @@ async def audio_socket(websocket: WebSocket) -> None:
}},
)
continue
session = await create_voice_session(websocket, request.session_id or None)
try:
session = await create_voice_session(
websocket, request.session_id or None,
server_vad=request.turn_detection == "server",
)
except Exception:
logger.exception("Failed to initialize voice session.")
await report("SESSION_START_FAILED", "Voice session could not be initialized.")
continue
server_vad = request.turn_detection == "server"
max_audio_chunk_bytes = min(
MAX_AUDIO_CHUNK_BYTES, session.max_input_chunk_bytes,
) if server_vad else MAX_AUDIO_CHUNK_BYTES
if max_audio_chunk_bytes < 2:
await report("INVALID_CONFIGURATION", "Audio input limit must allow a complete PCM frame.")
break
await send_json(websocket, SessionReady(
protocol_version=PROTOCOL_VERSION, session_id=session.session_id,
audio=CANONICAL_AUDIO_FORMAT,
output_audio=session.output_audio_format,
turn_detection=request.turn_detection,
max_audio_chunk_bytes=max_audio_chunk_bytes,
))
continue
if session is None:
@@ -246,6 +271,43 @@ async def audio_socket(websocket: WebSocket) -> None:
await report("SESSION_ALREADY_CLOSED", "This session is already closed.")
continue
if server_vad and event_type in {"audio.start", "audio.chunk", "audio.stop"}:
await report("TURN_CONTROLLED_BY_VAD", "This session uses server-side turn detection.")
continue
if event_type == "audio.input":
if not server_vad:
await report("INVALID_INPUT_MODE", "Continuous audio requires server-side turn detection.")
continue
try:
request = InputAudioChunk.model_validate(payload)
if pending_audio_chunk is not None:
raise ValueError("Previous audio chunk is still pending PCM data.")
if request.sequence_number != expected_input_sequence:
raise ValueError(f"Expected sequence_number={expected_input_sequence}.")
pending_audio_chunk = request
except (ValidationError, ValueError) as exc:
await report(
"AUDIO_SEQUENCE_ERROR", str(exc),
details={"expected_sequence_number": expected_input_sequence},
)
continue
if event_type == "audio.input.end":
if not server_vad:
await report("INVALID_INPUT_MODE", "Continuous audio requires server-side turn detection.")
continue
try:
InputAudioEnd.model_validate(payload)
if pending_audio_chunk is not None:
raise ValueError("An audio chunk is still pending PCM data.")
await session.finish_audio_input()
except VoiceSessionError as exc:
await report(exc.code, str(exc), turn_id=exc.turn_id, retryable=exc.retryable)
except (ValidationError, ValueError) as exc:
await report("INVALID_MESSAGE", str(exc))
continue
if event_type == "audio.start":
try:
turn_id = await session.start_turn()

16
src/voice_assistant/audio_gateway/app/protocol.py
View File

@@ -20,6 +20,7 @@ class SessionStart(BaseModel):
type: Literal["session.start"] = "session.start"
session_id: str | None = None
turn_detection: Literal["client", "server"] = "client"
audio: AudioFormat = Field(
default_factory=lambda: CANONICAL_AUDIO_FORMAT.model_copy()
@@ -54,6 +55,19 @@ class AudioChunk(BaseModel):
sequence_number: int
class InputAudioChunk(BaseModel):
"""Metadata for continuous PCM, sequenced across the entire session."""
type: Literal["audio.input"] = "audio.input"
sequence_number: int = Field(ge=0, strict=True)
class InputAudioEnd(BaseModel):
"""Normal end of a finite continuous input stream."""
type: Literal["audio.input.end"] = "audio.input.end"
class AudioStop(BaseModel):
"""Request that finishes audio input for the active turn."""
@@ -82,6 +96,8 @@ class SessionReady(BaseModel):
session_id: str
audio: AudioFormat
output_audio: AudioFormat
turn_detection: Literal["client", "server"] = "client"
max_audio_chunk_bytes: int = Field(default=65536, ge=2)
class TurnStarted(BaseModel):

4
src/voice_assistant/audio_gateway/web/index.html
View File

@@ -72,7 +72,7 @@ const audioBands=new Float32Array(3);
let voiceController=null;
const activeStates=new Set([
'connecting','listening','starting','recording','flushing',
'connecting','listening','recording',
'processing','speaking','waiting-playback'
]);
@@ -107,9 +107,7 @@ const assistantIntegration={
const messages={
connecting:'Conectando ao AgentVoice...',
listening:'Estou ouvindo. Pode falar.',
starting:'Iniciando reconhecimento...',
recording:'Estou ouvindo sua pergunta...',
flushing:'Finalizando captura de áudio...',
processing:'Preparando sua resposta...',
speaking:'Reproduzindo resposta...',
'waiting-playback':'Aguardando fim da reprodução.'

19
src/voice_assistant/audio_gateway/web/pcm-worklet.js
View File

@@ -16,10 +16,19 @@ class PCMInputProcessor extends AudioWorkletProcessor {
this.port.onmessage = ({ data }) => {
switch (data?.type) {
case "start":
case "start": {
const chunkSamples = data.chunkSamples ?? 1600;
if (!Number.isInteger(chunkSamples) || chunkSamples < 1 || chunkSamples > 1600) {
this.capturing = false;
this.reset();
this.port.postMessage({ type: "error", message: "Tamanho de captura inválido." });
return;
}
this.chunkSamples = chunkSamples;
this.reset();
this.capturing = true;
break;
}
case "flush":
this.capturing = false;
@@ -44,14 +53,16 @@ class PCMInputProcessor extends AudioWorkletProcessor {
emitPCM() {
if (!this.output.length) return;
const pcm = new Int16Array(this.output);
const pcm = new ArrayBuffer(this.output.length * 2);
const view = new DataView(pcm);
this.output.forEach((sample, index) => view.setInt16(index * 2, sample, true));
this.output = [];
this.port.postMessage({
type: "pcm",
sequence: this.sequence++,
buffer: pcm.buffer
}, [pcm.buffer]);
buffer: pcm
}, [pcm]);
}
flush() {

92
src/voice_assistant/audio_gateway/web/vad.js
View File

@@ -1,92 +0,0 @@
export class VoiceActivityDetector {
constructor({
onSpeechStart,
onSpeechEnd,
minSpeechMs = 250,
silenceMs = 800,
threshold = 0.012,
noiseMultiplier = 3,
} = {}) {
this.onSpeechStart = onSpeechStart;
this.onSpeechEnd = onSpeechEnd;
this.minSpeechMs = minSpeechMs;
this.silenceMs = silenceMs;
this.threshold = threshold;
this.noiseMultiplier = noiseMultiplier;
this.noiseFloor = 0.003;
this.speaking = false;
this.speechStartedAt = null;
this.lastVoiceAt = null;
this.active = false;
}
start() {
this.active = true;
this.reset();
}
stop() {
this.active = false;
this.reset();
}
reset() {
this.speaking = false;
this.speechStartedAt = null;
this.lastVoiceAt = null;
}
process(rms, now = performance.now()) {
if (!this.active) return;
if (!Number.isFinite(rms) || rms < 0) return;
const threshold = Math.max(
this.threshold,
this.noiseFloor * this.noiseMultiplier
);
const voiced = rms >= threshold;
if (!this.speaking) {
// Update noise estimation only outside detected speech.
if (!voiced) {
this.noiseFloor =
this.noiseFloor * 0.98 + rms * 0.02;
}
if (voiced) {
if (this.speechStartedAt === null) {
this.speechStartedAt = now;
}
if (now - this.speechStartedAt >= this.minSpeechMs) {
this.speaking = true;
this.lastVoiceAt = now;
this.onSpeechStart?.();
}
} else {
this.speechStartedAt = null;
}
return;
}
if (voiced) {
this.lastVoiceAt = now;
return;
}
if (
this.lastVoiceAt !== null &&
now - this.lastVoiceAt >= this.silenceMs
) {
this.speaking = false;
this.onSpeechEnd?.();
this.reset();
}
}
}

833
src/voice_assistant/audio_gateway/web/voice-session.js
View File

@@ -1,5 +1,3 @@
import { VoiceActivityDetector } from "./vad.js";
import { PCMPlayer } from "./pcm-player.js";
export class VoiceSessionController {
@@ -8,78 +6,45 @@ export class VoiceSessionController {
onTranscript = () => {},
onAgentText = () => {},
onError = () => {},
silenceMs = 800,
prebufferChunks = 4,
maxPendingChunks = 100,
maxBufferedBytes = 256 * 1024,
handshakeTimeoutMs = 10000,
} = {}) {
if (!Number.isSafeInteger(maxBufferedBytes) || maxBufferedBytes < 2) {
throw new Error("Limite de envio de áudio inválido.");
}
if (!Number.isFinite(handshakeTimeoutMs) || handshakeTimeoutMs <= 0) {
throw new Error("Prazo de conexão inválido.");
}
this.onStateChange = onStateChange;
this.onTranscript = onTranscript;
this.onAgentText = onAgentText;
this.onError = onError;
this.prebufferChunks = prebufferChunks;
this.maxPendingChunks = maxPendingChunks;
this.maxBufferedBytes = maxBufferedBytes;
this.handshakeTimeoutMs = handshakeTimeoutMs;
this.state = "closed";
this.ws = null;
this.stream = null;
this.context = null;
this.analyser = null;
this.worklet = null;
this.source = null;
this.silentGain = null;
this.turnId = null;
this.sequence = 0;
this.prebuffer = [];
this.pending = [];
this.stopRequested = false;
this.frameId = 0;
this.rmsSamples = null;
this.vad = new VoiceActivityDetector({
silenceMs,
onSpeechStart: () => this.beginTurn(),
onSpeechEnd: () => this.finishSpeech(),
});
this.player = null;
this.playbackPending = Promise.resolve();
this.ready = false;
this.sequence = 0;
this.turnId = null;
this.latestTurnId = 0;
this.gatewayTurnEnded = false;
this.playbackGeneration = 0;
this.bargeInPending = false;
this.bargeInBuffer = [];
this.maxBargeInChunks = 100;
this.bargeInSpeechEnded = false;
this.interruptedTurnId = null;
this.bargeInCaptureStarted = false;
this.bargeInVad = new VoiceActivityDetector({
minSpeechMs: 350,
silenceMs: 800,
threshold: 0.025,
noiseMultiplier: 4,
onSpeechStart: () => this.requestBargeIn(),
onSpeechEnd: () => this.finishBargeInSpeech(),
});
this.connectionGeneration = 0;
this.playbackPending = Promise.resolve();
this.handshakeTimer = null;
this.maxAudioChunkBytes = 0;
}
getPlaybackAnalyser() {
return this.player?.getAnalyser() ?? null;
return this.player?.getAnalyser() ?? null;
}
setState(state) {
console.debug(
`[AgentVoice][STATE] ${this.state} -> ${state}`,
{
turnId: this.turnId,
playerFinished: this.player?.finished,
bargeInPending: this.bargeInPending,
bargeInRecording: this.bargeInRecording,
}
);
this.state = state;
this.onStateChange(state);
}
@@ -91,688 +56,234 @@ export class VoiceSessionController {
this.ws.send(JSON.stringify(event));
}
async start({ stream, audioContext, analyser }) {
if (this.state !== "closed") {
throw new Error("Sessão já iniciada.");
}
this.stream = stream;
async start({ stream, audioContext }) {
if (this.state !== "closed") throw new Error("Sessão já iniciada.");
const generation = ++this.connectionGeneration;
this.context = audioContext;
this.analyser = analyser;
this.ready = false;
this.sequence = 0;
this.turnId = null;
this.latestTurnId = 0;
this.player = new PCMPlayer({
audioContext,
onPlaybackStart: () => {
if (
this.state === "processing" ||
this.state === "speaking"
) {
this.setState("speaking");
this.bargeInVad.start();
this.monitorVoice();
}
},
onPlaybackStart: () => {
if (this.ready && this.turnId !== null) this.setState("speaking");
},
});
this.rmsSamples = new Float32Array(analyser.fftSize);
this.setState("connecting");
try {
await this.context.audioWorklet.addModule(
"/web/pcm-worklet.js"
);
this.source = this.context.createMediaStreamSource(stream);
this.worklet = new AudioWorkletNode(
this.context,
"pcm-input"
);
this.silentGain = this.context.createGain();
await audioContext.audioWorklet.addModule("/web/pcm-worklet.js");
if (generation !== this.connectionGeneration) return;
this.source = audioContext.createMediaStreamSource(stream);
this.worklet = new AudioWorkletNode(audioContext, "pcm-input");
this.silentGain = audioContext.createGain();
this.silentGain.gain.value = 0;
this.source.connect(this.worklet);
this.worklet.connect(this.silentGain);
this.silentGain.connect(this.context.destination);
this.silentGain.connect(audioContext.destination);
this.worklet.port.onmessage = ({ data }) => {
this.handleWorkletMessage(data);
if (generation !== this.connectionGeneration) return;
if (data?.type === "pcm") this.handlePCM(data.buffer);
if (data?.type === "error") this.fail(new Error(data.message));
};
const scheme = location.protocol === "https:" ? "wss" : "ws";
const socket = new WebSocket(
`${scheme}://${location.host}/ws/audio`
);
const socket = new WebSocket(`${scheme}://${location.host}/ws/audio`);
this.ws = socket;
socket.binaryType = "arraybuffer";
socket.onopen = () => {
if (this.ws !== socket) return;
this.send({
type: "session.start",
audio: {
encoding: "pcm_s16le",
sample_rate: 16000,
channels: 1,
},
});
try {
this.send({
type: "session.start",
turn_detection: "server",
audio: { encoding: "pcm_s16le", sample_rate: 16000, channels: 1 },
});
} catch (error) {
this.fail(error);
}
};
socket.onmessage = ({ data }) => {
if (this.ws !== socket) return;
if (typeof data === "string") {
try {
this.handleGatewayEvent(JSON.parse(data));
} catch (error) {
this.fail(error);
}
} else if (data instanceof ArrayBuffer) {
this.handleAudioChunk(data);
try {
if (typeof data === "string") this.handleGatewayEvent(JSON.parse(data));
else if (data instanceof ArrayBuffer) this.handleAudioChunk(data);
else throw new Error("Resposta de áudio inválida.");
} catch (error) {
this.fail(error);
}
};
};
socket.onerror = () => {
if (this.ws === socket) {
this.fail(new Error("Falha na conexão WebSocket."));
}
if (this.ws === socket) this.fail(new Error("Falha na conexão WebSocket."));
};
socket.onclose = () => {
if (this.ws === socket && this.state !== "closed") {
this.fail(new Error("Conexão WebSocket encerrada."));
}
if (this.ws === socket) this.fail(new Error("Conexão WebSocket encerrada."));
};
this.handshakeTimer = setTimeout(() => {
if (this.ws === socket && !this.ready) {
this.fail(new Error("O servidor não confirmou o início da conversa."));
}
}, this.handshakeTimeoutMs);
} catch (error) {
if (generation !== this.connectionGeneration) return;
this.close();
throw error;
}
}
startListening() {
this.playbackGeneration++;
this.gatewayTurnEnded = false;
this.player?.reset();
this.playbackPending = Promise.resolve();
this.bargeInVad.stop();
this.bargeInPending = false;
this.bargeInSpeechEnded = false;
this.interruptedTurnId = null;
this.bargeInBuffer = [];
this.turnId = null;
this.sequence = 0;
this.prebuffer = [];
this.pending = [];
this.stopRequested = false;
this.worklet.port.postMessage({ type: "start" });
this.vad.start();
this.setState("listening");
this.monitorVoice();
}
monitorVoice() {
cancelAnimationFrame(this.frameId);
const tick = (now) => {
const normalListening = [
"listening",
"starting",
"recording"
].includes(this.state);
const bargeInListening = [
"processing",
"speaking",
"waiting-playback",
"interrupting"
].includes(this.state) || this.bargeInRecording;
if (!normalListening && !bargeInListening) {
return;
}
this.analyser.getFloatTimeDomainData(this.rmsSamples);
let power = 0;
for (const sample of this.rmsSamples) {
power += sample * sample;
}
const rms = Math.sqrt(
power / this.rmsSamples.length
);
// Normal speech detection.
if (
normalListening &&
!this.bargeInRecording &&
!this.stopRequested
) {
this.vad.process(rms, now);
}
// Continue tracking speech initiated during TTS.
if (
bargeInListening &&
this.bargeInVad.active
) {
this.bargeInVad.process(rms, now);
}
this.frameId = requestAnimationFrame(tick);
};
this.frameId = requestAnimationFrame(tick);
}
beginTurn() {
if (this.state !== "listening") return;
this.pending = [...this.prebuffer];
this.setState("starting");
try {
this.send({ type: "audio.start" });
} catch (error) {
this.fail(error);
}
}
handleWorkletMessage(data) {
if (data?.type === "pcm") {
this.handlePCM(data.buffer);
} else if (data?.type === "flushed") {
this.handleFlushed();
}
}
handlePCM(buffer) {
if (this.state === "listening") {
this.prebuffer.push(buffer);
if (this.prebuffer.length > this.prebufferChunks) {
this.prebuffer.shift();
}
return;
}
if (
this.state === "processing" ||
this.state === "speaking" ||
this.state === "waiting-playback" ||
this.state === "interrupting"
) {
this.bargeInBuffer.push(buffer);
if (this.bargeInBuffer.length > this.maxBargeInChunks) {
this.bargeInBuffer.shift();
}
return;
}
if (this.state === "starting") {
this.pending.push(buffer);
if (this.pending.length > this.maxPendingChunks) {
this.fail(new Error("Buffer de início de turno excedido."));
}
return;
}
if (this.state === "recording" ||
this.state === "flushing") {
this.sendPCM(buffer);
}
}
sendPCM(buffer) {
if (this.turnId === null) return;
if (this.ws?.readyState !== WebSocket.OPEN) return;
this.send({
type: "audio.chunk",
turn_id: this.turnId,
sequence_number: this.sequence++,
});
this.ws.send(buffer);
}
finishSpeech() {
if (this.stopRequested) return;
if (this.state !== "starting" &&
this.state !== "recording") return;
this.stopRequested = true;
this.vad.stop();
if (this.state === "recording") {
this.flushTurn();
}
// If still starting, flush after turn.started.
}
flushTurn() {
if (this.state !== "recording") return;
this.setState("flushing");
this.worklet.port.postMessage({ type: "flush" });
}
handleFlushed() {
if (this.state !== "flushing") return;
this.send({
type: "audio.stop",
turn_id: this.turnId,
});
this.setState("processing");
// Resume local PCM capture for a possible interruption.
// Audio will remain buffered until a new turn is accepted.
this.worklet.port.postMessage({ type: "start" });
// The microphone remains available for barge-in.
// Detection starts when TTS playback begins.
this.monitorVoice();
}
handleAudioChunk(buffer) {
if (
this.state === "closed" ||
this.state === "interrupting" ||
this.bargeInPending ||
this.turnId === null ||
this.gatewayTurnEnded
) {
return;
}
console.debug("[AgentVoice][PCM recebido]", {
bytes: buffer.byteLength,
turnId: this.turnId,
state: this.state,
generation: this.playbackGeneration,
finished: this.player?.finished,
gatewayTurnEnded: this.gatewayTurnEnded,
});
const generation = this.playbackGeneration;
this.playbackPending = this.playbackPending
.then(async () => {
if (generation !== this.playbackGeneration) {
return;
}
if (
this.state === "interrupting" ||
this.bargeInPending
) {
return;
}
console.debug("[AgentVoice][PCM enqueue]", {
turnId: this.turnId,
state: this.state,
generation,
currentGeneration: this.playbackGeneration,
finished: this.player?.finished,
});
await this.player.enqueue(buffer);
})
.catch((error) => {
// An obsolete playback must never terminate the session.
if (generation !== this.playbackGeneration) {
return;
}
if (
this.state === "interrupting" ||
this.bargeInPending
) {
return;
}
this.fail(error);
});
}
async finishPlaybackAndResume() {
const generation = this.playbackGeneration;
if (!this.ready || this.state === "closed") return;
try {
await this.playbackPending;
if (generation !== this.playbackGeneration) return;
if (this.state === "closed") return;
this.player.finish();
await this.player.waitForCompletion();
if (generation !== this.playbackGeneration) return;
if (this.state === "closed") return;
if (!this.gatewayTurnEnded) return;
this.startListening();
} catch (error) {
if (generation === this.playbackGeneration) {
this.fail(error);
}
}
}
requestBargeIn() {
if (this.bargeInPending) return;
if (
this.state !== "speaking" &&
this.state !== "waiting-playback"
) {
return;
}
if (this.turnId === null) return;
const interruptedTurnId = this.turnId;
this.bargeInPending = true;
this.bargeInSpeechEnded = false;
this.bargeInCaptureStarted = true;
this.interruptedTurnId = interruptedTurnId;
// Stop local playback immediately.
this.playbackGeneration++;
this.player?.stop();
this.playbackPending = Promise.resolve();
// Prevent normal playback completion from resuming listening.
this.gatewayTurnEnded = false;
// The normal VAD must not start a competing turn.
this.vad.stop();
this.setState("interrupting");
try {
this.send({
type: "interrupt",
turn_id: interruptedTurnId,
});
if (!(buffer instanceof ArrayBuffer) || !buffer.byteLength || buffer.byteLength % 2) {
throw new Error("Áudio capturado inválido.");
}
if (buffer.byteLength > this.maxAudioChunkBytes) {
throw new Error("O bloco de áudio excede o limite do servidor.");
}
if ((this.ws?.bufferedAmount ?? 0) + buffer.byteLength > this.maxBufferedBytes) {
throw new Error("A conexão não está acompanhando o envio de áudio.");
}
this.send({ type: "audio.input", sequence_number: this.sequence });
this.ws.send(buffer);
this.sequence++;
} catch (error) {
this.fail(error);
}
}
finishBargeInSpeech() {
if (!this.bargeInPending && !this.bargeInRecording) {
return;
}
invalidatePlayback() {
this.playbackGeneration++;
this.player?.stop();
this.playbackPending = Promise.resolve();
this.gatewayTurnEnded = false;
}
this.bargeInSpeechEnded = true;
handleAudioChunk(buffer) {
if (!this.ready || this.turnId === null || this.gatewayTurnEnded) return;
const generation = this.playbackGeneration;
this.playbackPending = this.playbackPending.then(async () => {
if (generation !== this.playbackGeneration || !this.ready) return;
await this.player.enqueue(buffer);
}).catch((error) => {
if (generation === this.playbackGeneration && this.ready) this.fail(error);
});
}
if (
this.bargeInRecording &&
this.state === "recording"
) {
this.bargeInRecording = false;
this.bargeInVad.stop();
this.stopRequested = true;
this.flushTurn();
async finishPlayback(generation, turnId) {
try {
await this.playbackPending;
if (generation !== this.playbackGeneration || !this.ready) return;
this.player.finish();
await this.player.waitForCompletion();
if (generation !== this.playbackGeneration || !this.ready || this.turnId !== turnId) return;
this.turnId = null;
this.setState("listening");
} catch (error) {
if (generation === this.playbackGeneration && this.ready) this.fail(error);
}
}
handleGatewayEvent(event) {
switch (event.type) {
case "session.ready":
this.player.configure(event.output_audio);
this.startListening();
break;
// Debug logging for all events received from the Gateway.
console.debug("[AgentVoice][Gateway]", {
type: event.type,
turnId: event.turn_id,
currentTurnId: this.turnId,
state: this.state,
playbackGeneration: this.playbackGeneration,
playerFinished: this.player?.finished,
playerPlaying: this.player?.playing,
bargeInPending: this.bargeInPending,
bargeInRecording: this.bargeInRecording,
text: event.type === "stt.final"
? event.text
: event.type === "agent.response"
? event.text
: undefined,
error: event.type === "error"
? { code: event.code, message: event.message }
: undefined,
});
// Debug logging for all events received from the Gateway.
case "turn.started":
if (this.state !== "starting") return;
// Prepare playback for the newly accepted turn.
// The previous TTS may have been stopped by barge-in.
this.playbackGeneration++;
this.player?.reset();
this.playbackPending = Promise.resolve();
this.gatewayTurnEnded = false;
this.turnId = event.turn_id;
this.sequence = 0;
this.setState("recording");
for (const buffer of this.pending) {
this.sendPCM(buffer);
}
this.pending = [];
if (this.bargeInPending) {
this.bargeInPending = false;
this.bargeInCaptureStarted = false;
if (this.bargeInSpeechEnded) {
// User finished speaking before turn.started.
this.bargeInRecording = false;
this.bargeInVad.stop();
this.stopRequested = true;
} else {
// User is still speaking.
// Preserve the active barge-in VAD state.
this.bargeInRecording = true;
this.stopRequested = false;
}
this.bargeInSpeechEnded = false;
if (this.state === "closed") return;
if (event.type === "session.ready") {
if (this.ready) throw new Error("Confirmação de sessão duplicada.");
if (
event.turn_detection !== "server" || event.audio?.encoding !== "pcm_s16le" ||
event.audio.sample_rate !== 16000 || event.audio.channels !== 1 ||
!Number.isInteger(event.max_audio_chunk_bytes) || event.max_audio_chunk_bytes < 2
) {
throw new Error("O servidor não oferece suporte a esta versão do cliente.");
}
if (this.stopRequested) {
this.flushTurn();
}
break;
this.player.configure(event.output_audio);
this.maxAudioChunkBytes = event.max_audio_chunk_bytes;
this.ready = true;
clearTimeout(this.handshakeTimer);
this.handshakeTimer = null;
this.worklet.port.postMessage({
type: "start", chunkSamples: Math.min(1600, Math.floor(this.maxAudioChunkBytes / 2)),
});
this.setState("listening");
return;
}
if (event.type === "error") {
this.fail(new Error(`${event.code}: ${event.message}`));
return;
}
if (event.type === "session.ended") {
this.close();
return;
}
if (!this.ready) return;
if (event.type === "turn.started") {
if (!Number.isSafeInteger(event.turn_id) || event.turn_id <= this.latestTurnId) return;
this.invalidatePlayback();
this.player.reset();
this.latestTurnId = event.turn_id;
this.turnId = event.turn_id;
this.setState("recording");
return;
}
if (event.turn_id !== this.turnId) return;
switch (event.type) {
case "stt.partial":
case "stt.final":
if (event.turn_id === this.turnId) {
this.onTranscript(event.text, event.type === "stt.final");
}
this.onTranscript(event.text, event.type === "stt.final");
if (event.type === "stt.final") this.setState("processing");
break;
case "agent.delta":
if (event.turn_id === this.turnId) {
this.onAgentText(event.text, false);
}
break;
case "agent.response":
if (event.turn_id === this.turnId) {
this.onAgentText(event.text, true);
}
this.onAgentText(event.text, event.type === "agent.response");
break;
case "turn.ended":
if (
event.turn_id === this.turnId &&
!this.bargeInPending
) {
this.gatewayTurnEnded = true;
this.setState("waiting-playback");
void this.finishPlaybackAndResume();
}
break;
case "interrupted":
if (
!this.bargeInPending ||
event.turn_id !== this.interruptedTurnId
) {
return;
}
this.interruptedTurnId = null;
this.invalidatePlayback();
this.turnId = null;
this.sequence = 0;
this.pending = [...this.bargeInBuffer];
this.bargeInBuffer = [];
this.stopRequested = this.bargeInSpeechEnded;
this.setState("starting");
try {
this.send({ type: "audio.start" });
} catch (error) {
this.fail(error);
}
this.setState("listening");
break;
case "error":
if (
event.code === "INVALID_TURN" &&
this.bargeInPending &&
event.turn_id === this.interruptedTurnId
) {
// The old turn completed before interrupt was processed.
// Its semantic state is already released by the Gateway.
this.interruptedTurnId = null;
this.turnId = null;
this.sequence = 0;
this.pending = [...this.bargeInBuffer];
this.bargeInBuffer = [];
this.stopRequested = this.bargeInSpeechEnded;
this.gatewayTurnEnded = false;
this.setState("starting");
try {
this.send({ type: "audio.start" });
} catch (error) {
this.fail(error);
}
break;
}
this.fail(
new Error(`${event.code}: ${event.message}`)
);
break;
case "session.ended":
this.close();
case "turn.ended":
if (this.gatewayTurnEnded) return;
this.gatewayTurnEnded = true;
this.setState("waiting-playback");
void this.finishPlayback(this.playbackGeneration, this.turnId);
break;
}
}
fail(error) {
this.onError(error);
this.close();
if (this.state === "closed") return;
try {
this.onError(error);
} finally {
this.close();
}
}
close() {
if (this.state === "closed") return;
this.playbackGeneration++;
this.player?.stop();
this.vad.stop();
this.bargeInVad.stop();
this.bargeInPending = false;
this.bargeInSpeechEnded = false;
this.bargeInCaptureStarted = false;
this.interruptedTurnId = null;
this.bargeInBuffer = [];
cancelAnimationFrame(this.frameId);
this.connectionGeneration++;
this.ready = false;
clearTimeout(this.handshakeTimer);
this.handshakeTimer = null;
this.invalidatePlayback();
if (this.worklet) {
this.worklet.port.postMessage({ type: "stop" });
this.worklet.port.onmessage = null;
this.worklet.disconnect();
}
this.source?.disconnect();
this.silentGain?.disconnect();
const socket = this.ws;
this.ws = null;
if (socket) {
socket.onclose = null;
socket.onerror = null;
socket.onmessage = null;
if (socket.readyState === WebSocket.OPEN) {
if (this.turnId === null) {
socket.send(JSON.stringify({ type: "session.end" }));
} else {
socket.send(JSON.stringify({
type: "interrupt",
turn_id: this.turnId,
}));
}
socket.onopen = socket.onmessage = socket.onerror = socket.onclose = null;
try {
if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: "session.end" }));
} catch {
// Local cleanup must complete even when the transport has already failed.
} finally {
socket.close();
}
socket.close();
}
this.worklet = null;
this.source = null;
this.silentGain = null;
this.worklet = this.source = this.silentGain = null;
this.turnId = null;
this.prebuffer = [];
this.pending = [];
this.setState("closed");
}
}