feat: migra cliente web para áudio contínuo com VAD no backend
This commit is contained in:
alex alves
48
tests/js/pcm-worklet.test.mjs
Normal file
48
tests/js/pcm-worklet.test.mjs
Normal file
@@ -0,0 +1,48 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { test } from "node:test";
|
||||
import vm from "node:vm";
|
||||
|
||||
function processor() {
|
||||
let Processor;
|
||||
class Base {
|
||||
constructor() {
|
||||
this.port = { sent: [], postMessage(data) { this.sent.push(data); } };
|
||||
}
|
||||
}
|
||||
vm.runInNewContext(readFileSync(new URL("../../src/voice_assistant/audio_gateway/web/pcm-worklet.js", import.meta.url), "utf8"), {
|
||||
AudioWorkletProcessor: Base, sampleRate: 16000,
|
||||
registerProcessor(name, constructor) { assert.equal(name, "pcm-input"); Processor = constructor; },
|
||||
});
|
||||
return new Processor();
|
||||
}
|
||||
|
||||
test("negotiated capture emits complete little-endian PCM16 frames", () => {
|
||||
const input = processor();
|
||||
input.port.onmessage({ data: { type: "start", chunkSamples: 2 } });
|
||||
input.process([[new Float32Array([-1, 0, 1])]]);
|
||||
const chunk = input.port.sent[0];
|
||||
assert.equal(chunk.type, "pcm");
|
||||
assert.equal(chunk.buffer.byteLength, 4);
|
||||
const samples = new DataView(chunk.buffer);
|
||||
assert.equal(samples.getInt16(0, true), -32768);
|
||||
assert.equal(samples.getInt16(2, true), 0);
|
||||
});
|
||||
|
||||
test("capture continues across process callbacks until stop", () => {
|
||||
const input = processor();
|
||||
input.port.onmessage({ data: { type: "start", chunkSamples: 2 } });
|
||||
input.process([[new Float32Array([0, 0, 0])]]);
|
||||
input.process([[new Float32Array([0, 0])]]);
|
||||
assert.equal(input.port.sent.filter((event) => event.type === "pcm").length, 2);
|
||||
input.port.onmessage({ data: { type: "stop" } });
|
||||
input.process([[new Float32Array([0, 0, 0])]]);
|
||||
assert.equal(input.port.sent.length, 2);
|
||||
});
|
||||
|
||||
test("invalid capture window reports an error", () => {
|
||||
const input = processor();
|
||||
input.port.onmessage({ data: { type: "start", chunkSamples: 0 } });
|
||||
assert.equal(input.port.sent[0].type, "error");
|
||||
assert.equal(input.capturing, false);
|
||||
});
|
||||
@@ -1,83 +0,0 @@
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { VoiceActivityDetector } from "../../src/voice_assistant/audio_gateway/web/vad.js";
|
||||
|
||||
test("detects speech start after minimum duration", () => {
|
||||
let starts = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 250,
|
||||
onSpeechStart: () => starts++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
|
||||
vad.process(0.04, 0);
|
||||
vad.process(0.04, 100);
|
||||
|
||||
assert.equal(starts, 0);
|
||||
|
||||
vad.process(0.04, 260);
|
||||
|
||||
assert.equal(starts, 1);
|
||||
});
|
||||
|
||||
test("detects speech end after silence", () => {
|
||||
let ends = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 100,
|
||||
silenceMs: 800,
|
||||
onSpeechEnd: () => ends++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
|
||||
vad.process(0.04, 0);
|
||||
vad.process(0.04, 120);
|
||||
vad.process(0.04, 300);
|
||||
|
||||
vad.process(0.001, 600);
|
||||
vad.process(0.001, 1000);
|
||||
|
||||
assert.equal(ends, 0);
|
||||
|
||||
vad.process(0.001, 1101);
|
||||
|
||||
assert.equal(ends, 1);
|
||||
});
|
||||
|
||||
test("ignores short noise spikes", () => {
|
||||
let starts = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 250,
|
||||
onSpeechStart: () => starts++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
|
||||
vad.process(0.05, 0);
|
||||
vad.process(0.001, 80);
|
||||
vad.process(0.001, 300);
|
||||
|
||||
assert.equal(starts, 0);
|
||||
});
|
||||
|
||||
test("stop disables detection", () => {
|
||||
let starts = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 100,
|
||||
onSpeechStart: () => starts++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
vad.stop();
|
||||
|
||||
vad.process(0.05, 0);
|
||||
vad.process(0.05, 200);
|
||||
|
||||
assert.equal(starts, 0);
|
||||
});
|
||||
@@ -1,746 +1,270 @@
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { afterEach, test } from "node:test";
|
||||
import { VoiceSessionController } from "../../src/voice_assistant/audio_gateway/web/voice-session.js";
|
||||
|
||||
import { VoiceSessionController } from
|
||||
"../../src/voice_assistant/audio_gateway/web/voice-session.js";
|
||||
|
||||
const originalWebSocket = globalThis.WebSocket;
|
||||
|
||||
class FakeWebSocket {
|
||||
const controllers = [];
|
||||
class Node {
|
||||
constructor() { this.disconnected = false; }
|
||||
connect() {}
|
||||
disconnect() { this.disconnected = true; }
|
||||
}
|
||||
class Context {
|
||||
constructor() {
|
||||
this.destination = {};
|
||||
this.currentTime = 0;
|
||||
this.sources = [];
|
||||
this.audioWorklet = { addModule: async () => {} };
|
||||
this.nextResume = null;
|
||||
}
|
||||
createAnalyser() { return new Node(); }
|
||||
createGain() { return Object.assign(new Node(), { gain: { value: 1 } }); }
|
||||
createMediaStreamSource() { this.input = new Node(); return this.input; }
|
||||
async resume() { const pending = this.nextResume; this.nextResume = null; if (pending) await pending; }
|
||||
createBuffer(channels, length, sampleRate) {
|
||||
return { duration: length / sampleRate, copyToChannel(samples) { this.samples = samples; } };
|
||||
}
|
||||
createBufferSource() {
|
||||
const source = Object.assign(new Node(), {
|
||||
stopped: false, onended: null,
|
||||
start: () => this.sources.push(source),
|
||||
stop() { this.stopped = true; },
|
||||
end() { this.onended?.(); },
|
||||
});
|
||||
return source;
|
||||
}
|
||||
}
|
||||
class Worklet extends Node {
|
||||
constructor() {
|
||||
super();
|
||||
this.port = { messages: [], postMessage(data) { this.messages.push(data); }, onmessage: null };
|
||||
}
|
||||
}
|
||||
class Socket {
|
||||
static OPEN = 1;
|
||||
static CLOSED = 3;
|
||||
|
||||
constructor() {
|
||||
this.readyState = FakeWebSocket.OPEN;
|
||||
this.sent = [];
|
||||
this.closed = false;
|
||||
}
|
||||
|
||||
send(data) {
|
||||
this.sent.push(data);
|
||||
}
|
||||
|
||||
close() {
|
||||
this.closed = true;
|
||||
this.readyState = FakeWebSocket.CLOSED;
|
||||
}
|
||||
constructor(url) { this.url = url; this.readyState = 0; this.sent = []; this.bufferedAmount = 0; }
|
||||
send(data) { if (this.sendFailure) throw new Error("Transport failed"); this.sent.push(data); }
|
||||
open() { this.readyState = 1; this.onopen?.(); }
|
||||
close() { this.readyState = 3; this.onclose?.(); }
|
||||
receive(event) { this.onmessage?.({ data: JSON.stringify(event) }); }
|
||||
}
|
||||
|
||||
globalThis.WebSocket = FakeWebSocket;
|
||||
globalThis.cancelAnimationFrame = () => {};
|
||||
globalThis.requestAnimationFrame = () => 1;
|
||||
|
||||
|
||||
afterEach(() => {
|
||||
// Restore only state modified by individual tests.
|
||||
globalThis.WebSocket = Socket;
|
||||
globalThis.AudioWorkletNode = Worklet;
|
||||
globalThis.location = { protocol: "http:", host: "localhost:8001" };
|
||||
afterEach(() => { for (const controller of controllers.splice(0)) controller.close(); });
|
||||
const tick = () => new Promise((resolve) => setImmediate(resolve));
|
||||
const ready = (overrides = {}) => ({
|
||||
type: "session.ready", turn_detection: "server",
|
||||
audio: { encoding: "pcm_s16le", sample_rate: 16000, channels: 1 },
|
||||
output_audio: { encoding: "pcm_s16le", sample_rate: 48000, channels: 1 },
|
||||
max_audio_chunk_bytes: 65536, ...overrides,
|
||||
});
|
||||
|
||||
function makeController() {
|
||||
const states = [];
|
||||
const errors = [];
|
||||
const workletMessages = [];
|
||||
|
||||
function messages(socket) { return socket.sent.filter((data) => typeof data === "string").map(JSON.parse); }
|
||||
async function setup(options = {}, confirm = true) {
|
||||
const errors = [], states = [], transcripts = [], replies = [];
|
||||
const controller = new VoiceSessionController({
|
||||
onStateChange: (state) => states.push(state),
|
||||
onError: (error) => errors.push(error),
|
||||
onError: (error) => errors.push(error), onStateChange: (state) => states.push(state),
|
||||
onTranscript: (...args) => transcripts.push(args), onAgentText: (...args) => replies.push(args),
|
||||
...options,
|
||||
});
|
||||
|
||||
controller.ws = new FakeWebSocket();
|
||||
controller.worklet = {
|
||||
port: {
|
||||
postMessage: (data) => workletMessages.push(data),
|
||||
onmessage: null,
|
||||
},
|
||||
disconnect() {},
|
||||
};
|
||||
|
||||
controller.source = { disconnect() {} };
|
||||
controller.silentGain = { disconnect() {} };
|
||||
|
||||
return {
|
||||
controller,
|
||||
states,
|
||||
errors,
|
||||
workletMessages,
|
||||
};
|
||||
controllers.push(controller);
|
||||
const context = new Context();
|
||||
await controller.start({ stream: {}, audioContext: context });
|
||||
const socket = controller.ws;
|
||||
socket.open();
|
||||
if (confirm) socket.receive(ready());
|
||||
return { controller, context, socket, errors, states, transcripts, replies };
|
||||
}
|
||||
|
||||
test("speech detection requests a new turn", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.setState("listening");
|
||||
controller.beginTurn();
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[0]),
|
||||
{ type: "audio.start" }
|
||||
);
|
||||
});
|
||||
|
||||
test("PCM is buffered until turn.started", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.setState("listening");
|
||||
|
||||
test("requests server VAD and starts capture only after session.ready", async () => {
|
||||
const { controller, socket } = await setup({}, false);
|
||||
assert.equal(messages(socket)[0].turn_detection, "server");
|
||||
assert.deepEqual(controller.worklet.port.messages, []);
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
assert.equal(socket.sent.length, 1);
|
||||
socket.receive(ready());
|
||||
assert.deepEqual(controller.worklet.port.messages, [{ type: "start", chunkSamples: 1600 }]);
|
||||
assert.equal(controller.state, "listening");
|
||||
});
|
||||
|
||||
assert.equal(controller.prebuffer.length, 1);
|
||||
assert.equal(controller.ws.sent.length, 0);
|
||||
|
||||
controller.beginTurn();
|
||||
controller.handlePCM(new ArrayBuffer(6));
|
||||
|
||||
assert.equal(controller.pending.length, 2);
|
||||
assert.equal(controller.ws.sent.length, 1);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 1,
|
||||
for (const state of ["listening", "recording", "processing", "speaking", "waiting-playback"]) {
|
||||
test(`sends continuous PCM while ${state}, without turn metadata`, async () => {
|
||||
const { controller, socket } = await setup();
|
||||
controller.setState(state);
|
||||
const audio = new ArrayBuffer(4);
|
||||
controller.handlePCM(audio);
|
||||
assert.deepEqual(messages(socket).at(-1), { type: "audio.input", sequence_number: 0 });
|
||||
assert.equal(socket.sent.at(-1), audio);
|
||||
});
|
||||
}
|
||||
|
||||
test("global input sequence never resets when backend opens another turn", async () => {
|
||||
const { controller, socket } = await setup();
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
socket.receive({ type: "interrupted", turn_id: 1 });
|
||||
socket.receive({ type: "turn.started", turn_id: 2 });
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
assert.deepEqual(messages(socket).filter((m) => m.type === "audio.input").map((m) => m.sequence_number), [0, 1, 2]);
|
||||
assert.equal(messages(socket).some((m) => ["audio.start", "audio.stop", "interrupt"].includes(m.type)), false);
|
||||
assert.equal(controller.worklet.port.messages.length, 1);
|
||||
});
|
||||
|
||||
test("uses the negotiated chunk size for microphone capture", async () => {
|
||||
const { controller, socket } = await setup({}, false);
|
||||
socket.receive(ready({ max_audio_chunk_bytes: 1000 }));
|
||||
assert.deepEqual(controller.worklet.port.messages, [{ type: "start", chunkSamples: 500 }]);
|
||||
});
|
||||
|
||||
test("turn.ended waits for speakers without pausing microphone input", async () => {
|
||||
const { controller, context, socket } = await setup();
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
await controller.playbackPending;
|
||||
assert.equal(controller.state, "speaking");
|
||||
socket.receive({ type: "turn.ended", turn_id: 1 });
|
||||
await tick();
|
||||
assert.equal(controller.state, "waiting-playback");
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
assert.equal(messages(socket).at(-1).type, "audio.input");
|
||||
context.sources[0].end();
|
||||
await tick();
|
||||
assert.equal(controller.state, "listening");
|
||||
assert.equal(controller.worklet.port.messages.length, 1);
|
||||
});
|
||||
|
||||
test("a new backend turn stops old playback and ignores its pending completion", async () => {
|
||||
const { controller, context, socket } = await setup();
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
await controller.playbackPending;
|
||||
socket.receive({ type: "turn.ended", turn_id: 1 });
|
||||
await tick();
|
||||
socket.receive({ type: "turn.started", turn_id: 2 });
|
||||
await tick();
|
||||
assert.equal(context.sources[0].stopped, true);
|
||||
assert.equal(controller.turnId, 2);
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.pending.length, 0);
|
||||
assert.equal(controller.sequence, 2);
|
||||
assert.equal(controller.ws.sent.length, 5);
|
||||
});
|
||||
|
||||
test("PCM metadata precedes each binary frame", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.turnId = 7;
|
||||
controller.setState("recording");
|
||||
|
||||
const pcm = new ArrayBuffer(8);
|
||||
|
||||
controller.sendPCM(pcm);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[0]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 7,
|
||||
sequence_number: 0,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(controller.ws.sent[1], pcm);
|
||||
assert.equal(controller.sequence, 1);
|
||||
test("old queued PCM is discarded after playback generation changes", async () => {
|
||||
const { controller, context, socket } = await setup();
|
||||
let release;
|
||||
context.nextResume = new Promise((resolve) => { release = resolve; });
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
const obsolete = controller.playbackPending;
|
||||
await tick();
|
||||
socket.receive({ type: "turn.started", turn_id: 2 });
|
||||
release();
|
||||
await obsolete;
|
||||
assert.equal(context.sources.length, 0);
|
||||
controller.handleAudioChunk(new ArrayBuffer(4));
|
||||
await controller.playbackPending;
|
||||
assert.equal(context.sources.length, 1);
|
||||
});
|
||||
|
||||
test("speech end requests worklet flush", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("recording");
|
||||
|
||||
controller.finishSpeech();
|
||||
|
||||
assert.equal(controller.state, "flushing");
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
|
||||
assert.equal(controller.ws.sent.length, 0);
|
||||
test("a failure in obsolete playback does not close the new turn", async () => {
|
||||
const { controller, context, socket, errors } = await setup();
|
||||
let reject;
|
||||
context.nextResume = new Promise((resolve, fail) => { reject = fail; });
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
const obsolete = controller.playbackPending;
|
||||
await tick();
|
||||
socket.receive({ type: "turn.started", turn_id: 2 });
|
||||
reject(new Error("Old playback failed"));
|
||||
await obsolete;
|
||||
assert.equal(errors.length, 0);
|
||||
assert.equal(controller.turnId, 2);
|
||||
assert.equal(socket.readyState, Socket.OPEN);
|
||||
});
|
||||
|
||||
|
||||
test("audio.stop is sent after flushed", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("flushing");
|
||||
|
||||
controller.handleFlushed();
|
||||
|
||||
const messages = controller.ws.sent
|
||||
.filter((message) => typeof message === "string")
|
||||
.map((message) => JSON.parse(message));
|
||||
|
||||
assert.ok(
|
||||
messages.some(
|
||||
(message) =>
|
||||
message.type === "audio.stop" &&
|
||||
message.turn_id === 1
|
||||
)
|
||||
);
|
||||
|
||||
assert.equal(controller.state, "processing");
|
||||
test("stale turn events cannot change the current response", async () => {
|
||||
const { controller, socket, transcripts, replies } = await setup();
|
||||
socket.receive({ type: "turn.started", turn_id: 2 });
|
||||
socket.receive({ type: "stt.final", turn_id: 1, text: "old" });
|
||||
socket.receive({ type: "agent.response", turn_id: 1, text: "old" });
|
||||
socket.receive({ type: "interrupted", turn_id: 1 });
|
||||
socket.receive({ type: "turn.ended", turn_id: 1 });
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
assert.equal(controller.turnId, 2);
|
||||
assert.deepEqual(transcripts, []);
|
||||
assert.deepEqual(replies, []);
|
||||
socket.receive({ type: "stt.final", turn_id: 2, text: "new" });
|
||||
socket.receive({ type: "agent.response", turn_id: 2, text: "reply" });
|
||||
assert.deepEqual(transcripts, [["new", true]]);
|
||||
assert.deepEqual(replies, [["reply", true]]);
|
||||
});
|
||||
|
||||
test("interrupted returns to listening without stopping capture or starting a client turn", async () => {
|
||||
const { controller, socket } = await setup();
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
socket.receive({ type: "interrupted", turn_id: 1 });
|
||||
assert.equal(controller.state, "listening");
|
||||
assert.equal(controller.turnId, null);
|
||||
assert.deepEqual(messages(socket).map((m) => m.type), ["session.start"]);
|
||||
assert.equal(controller.worklet.port.messages.length, 1);
|
||||
});
|
||||
|
||||
test("speech ending before turn.started is handled", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.setState("listening");
|
||||
controller.beginTurn();
|
||||
controller.finishSpeech();
|
||||
|
||||
assert.equal(controller.stopRequested, true);
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 3,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "flushing");
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
});
|
||||
|
||||
test("closing a session releases resources", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
const socket = controller.ws;
|
||||
|
||||
controller.setState("listening");
|
||||
controller.close();
|
||||
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(socket.closed, true);
|
||||
assert.equal(controller.ws, null);
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "stop" }
|
||||
);
|
||||
});
|
||||
|
||||
test("gateway errors close the session", () => {
|
||||
const { controller, errors } = makeController();
|
||||
|
||||
controller.setState("processing");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "error",
|
||||
code: "STT_START_FAILED",
|
||||
message: "Provider unavailable",
|
||||
for (const metadata of [{ turn_detection: "client" }, { turn_detection: undefined }, { max_audio_chunk_bytes: 1 }]) {
|
||||
test(`rejects incompatible session metadata: ${JSON.stringify(metadata)}`, async () => {
|
||||
const { controller, socket, errors } = await setup({}, false);
|
||||
socket.receive(ready(metadata));
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(errors.length, 1);
|
||||
});
|
||||
}
|
||||
|
||||
test("backpressure closes the session before queuing more PCM", async () => {
|
||||
const { controller, socket, errors } = await setup({ maxBufferedBytes: 8 });
|
||||
socket.bufferedAmount = 8;
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(
|
||||
errors[0].message,
|
||||
/STT_START_FAILED/
|
||||
);
|
||||
assert.equal(messages(socket).some((m) => m.type === "audio.input"), false);
|
||||
});
|
||||
|
||||
|
||||
test("turn.ended waits for playback before listening", async () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let resolvePlayback;
|
||||
|
||||
controller.player = {
|
||||
finish() {},
|
||||
reset() {},
|
||||
stop() {},
|
||||
waitForCompletion() {
|
||||
return new Promise((resolve) => {
|
||||
resolvePlayback = resolve;
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("processing");
|
||||
|
||||
let resumed = false;
|
||||
|
||||
controller.startListening = () => {
|
||||
resumed = true;
|
||||
};
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.ended",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(resumed, false);
|
||||
|
||||
resolvePlayback();
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(resumed, true);
|
||||
test("invalid PCM closes the session without emitting metadata", async () => {
|
||||
const { controller, socket, errors } = await setup();
|
||||
controller.handlePCM(new ArrayBuffer(3));
|
||||
assert.equal(errors.length, 1);
|
||||
assert.equal(messages(socket).some((m) => m.type === "audio.input"), false);
|
||||
});
|
||||
|
||||
test("closing during playback prevents listening resume", async () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let resolvePlayback;
|
||||
|
||||
controller.player = {
|
||||
finish() {},
|
||||
reset() {},
|
||||
stop() {},
|
||||
waitForCompletion() {
|
||||
return new Promise((resolve) => {
|
||||
resolvePlayback = resolve;
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("processing");
|
||||
|
||||
let resumed = false;
|
||||
|
||||
controller.startListening = () => {
|
||||
resumed = true;
|
||||
};
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.ended",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
controller.close();
|
||||
|
||||
resolvePlayback();
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(resumed, false);
|
||||
assert.equal(controller.state, "closed");
|
||||
});
|
||||
|
||||
test("binary audio is forwarded to PCMPlayer", async () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
const received = [];
|
||||
|
||||
controller.player = {
|
||||
async enqueue(buffer) {
|
||||
received.push(buffer);
|
||||
},
|
||||
finish() {},
|
||||
reset() {},
|
||||
stop() {},
|
||||
waitForCompletion: async () => {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("processing");
|
||||
|
||||
const pcm = new ArrayBuffer(8);
|
||||
|
||||
controller.handleAudioChunk(pcm);
|
||||
|
||||
await controller.playbackPending;
|
||||
|
||||
assert.deepEqual(received, [pcm]);
|
||||
});
|
||||
|
||||
|
||||
|
||||
test("barge-in stops playback and starts a new turn", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let playbackStopped = false;
|
||||
|
||||
controller.player = {
|
||||
stop() {
|
||||
playbackStopped = true;
|
||||
},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
// Simulate user speech detected during TTS.
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(playbackStopped, true);
|
||||
assert.equal(controller.state, "interrupting");
|
||||
assert.equal(controller.bargeInPending, true);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[0]),
|
||||
{
|
||||
type: "interrupt",
|
||||
turn_id: 1,
|
||||
}
|
||||
);
|
||||
|
||||
// Gateway confirms the previous turn was cancelled.
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
assert.equal(controller.ws.closed, false);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[1]),
|
||||
{
|
||||
type: "audio.start",
|
||||
}
|
||||
);
|
||||
|
||||
// Gateway accepts the new turn.
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.turnId, 2);
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.bargeInPending, false);
|
||||
});
|
||||
|
||||
|
||||
|
||||
test("barge-in preserves PCM until new turn starts", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
const pcm1 = new ArrayBuffer(4);
|
||||
const pcm2 = new ArrayBuffer(6);
|
||||
const pcm3 = new ArrayBuffer(8);
|
||||
|
||||
// Audio captured before speech confirmation.
|
||||
controller.handlePCM(pcm1);
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
// Audio captured while interruption is pending.
|
||||
controller.handlePCM(pcm2);
|
||||
|
||||
assert.equal(controller.ws.sent.length, 1);
|
||||
assert.equal(controller.bargeInBuffer.length, 2);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
// Still no PCM sent to the old turn.
|
||||
assert.equal(controller.state, "starting");
|
||||
assert.equal(controller.ws.sent.length, 2);
|
||||
|
||||
// Additional speech arrives before turn.started.
|
||||
controller.handlePCM(pcm3);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
const sent = controller.ws.sent;
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[0]),
|
||||
{ type: "interrupt", turn_id: 1 }
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[1]),
|
||||
{ type: "audio.start" }
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[2]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 0,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(sent[3], pcm1);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[4]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 1,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(sent[5], pcm2);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[6]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 2,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(sent[7], pcm3);
|
||||
|
||||
assert.equal(controller.sequence, 3);
|
||||
assert.equal(controller.state, "recording");
|
||||
});
|
||||
|
||||
|
||||
test("barge-in finishes speech before new turn starts", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
const pcm = new ArrayBuffer(8);
|
||||
|
||||
controller.handlePCM(pcm);
|
||||
controller.requestBargeIn();
|
||||
|
||||
// User finishes speaking before the Gateway responds.
|
||||
controller.finishBargeInSpeech();
|
||||
|
||||
assert.equal(controller.bargeInSpeechEnded, true);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
// Buffered audio must be transmitted before flushing.
|
||||
assert.equal(controller.sequence, 1);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[2]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 0,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(controller.ws.sent[3], pcm);
|
||||
assert.equal(controller.state, "flushing");
|
||||
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
test("barge-in continues recording until speech ends", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.bargeInVad.start();
|
||||
controller.requestBargeIn();
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.bargeInRecording, true);
|
||||
assert.equal(controller.bargeInVad.active, true);
|
||||
|
||||
// Speech continues after the new turn starts.
|
||||
const pcm = new ArrayBuffer(8);
|
||||
controller.handlePCM(pcm);
|
||||
|
||||
assert.equal(controller.sequence, 1);
|
||||
|
||||
// Barge-in VAD detects the end of speech.
|
||||
controller.finishBargeInSpeech();
|
||||
|
||||
assert.equal(controller.bargeInRecording, false);
|
||||
assert.equal(controller.state, "flushing");
|
||||
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
|
||||
// Worklet confirms that remaining PCM was emitted.
|
||||
controller.handleFlushed();
|
||||
|
||||
assert.equal(controller.state, "processing");
|
||||
|
||||
const messages = controller.ws.sent
|
||||
.filter((message) => typeof message === "string")
|
||||
.map((message) => JSON.parse(message));
|
||||
|
||||
assert.ok(
|
||||
messages.some(
|
||||
(message) =>
|
||||
message.type === "audio.stop" &&
|
||||
message.turn_id === 2
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("late INVALID_TURN recovers barge-in session", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(controller.state, "interrupting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "error",
|
||||
code: "INVALID_TURN",
|
||||
message: "Turn 1 is no longer active.",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
assert.equal(controller.ws.closed, false);
|
||||
|
||||
const messages = controller.ws.sent
|
||||
.filter((message) => typeof message === "string")
|
||||
.map((message) => JSON.parse(message));
|
||||
|
||||
assert.deepEqual(
|
||||
messages.map((message) => message.type),
|
||||
["interrupt", "audio.start"]
|
||||
);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.turnId, 2);
|
||||
});
|
||||
|
||||
|
||||
test("late TTS chunk during barge-in does not close session", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let stopped = false;
|
||||
|
||||
controller.player = {
|
||||
stop() {
|
||||
stopped = true;
|
||||
},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(stopped, true);
|
||||
assert.equal(controller.state, "interrupting");
|
||||
|
||||
// A late PCM chunk from the interrupted TTS.
|
||||
test("close stops capture and playback even if session.end cannot be sent", async () => {
|
||||
const { controller, socket, context } = await setup();
|
||||
const worklet = controller.worklet;
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
|
||||
// The WebSocket session must remain alive.
|
||||
assert.equal(controller.ws.closed, false);
|
||||
assert.equal(controller.state, "interrupting");
|
||||
assert.equal(controller.bargeInPending, true);
|
||||
await controller.playbackPending;
|
||||
socket.sendFailure = true;
|
||||
controller.close();
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(worklet.disconnected, true);
|
||||
assert.deepEqual(worklet.port.messages.at(-1), { type: "stop" });
|
||||
assert.equal(context.sources[0].stopped, true);
|
||||
assert.equal(socket.readyState, Socket.CLOSED);
|
||||
});
|
||||
|
||||
|
||||
test("barge-in resets player before new TTS response", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let finished = false;
|
||||
let resetCount = 0;
|
||||
|
||||
controller.player = {
|
||||
stop() {
|
||||
finished = true;
|
||||
},
|
||||
|
||||
reset() {
|
||||
finished = false;
|
||||
resetCount++;
|
||||
},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(finished, true);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(finished, false);
|
||||
assert.equal(resetCount, 1);
|
||||
test("closing during worklet loading prevents orphan microphone graphs and sockets", async () => {
|
||||
let release;
|
||||
const context = new Context();
|
||||
context.audioWorklet.addModule = () => new Promise((resolve) => { release = resolve; });
|
||||
const controller = new VoiceSessionController();
|
||||
controllers.push(controller);
|
||||
const starting = controller.start({ stream: {}, audioContext: context });
|
||||
controller.close();
|
||||
release();
|
||||
await starting;
|
||||
assert.equal(controller.ws, null);
|
||||
assert.equal(context.input, undefined);
|
||||
});
|
||||
|
||||
|
||||
test("session.ready configures output before starting microphone capture", () => {
|
||||
const { controller } = makeController();
|
||||
const operations = [];
|
||||
const format = { encoding: "pcm_s16le", sample_rate: 24000, channels: 1 };
|
||||
controller.player = { configure: (received) => operations.push(received) };
|
||||
controller.startListening = () => operations.push("listen");
|
||||
controller.handleGatewayEvent({ type: "session.ready", output_audio: format });
|
||||
assert.deepEqual(operations, [format, "listen"]);
|
||||
test("missing session.ready times out and closes the connection", async () => {
|
||||
const { controller, errors } = await setup({ handshakeTimeoutMs: 5 }, false);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(errors.length, 1);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user