feat: migra cliente web para áudio contínuo com VAD no backend
This commit is contained in:
alex alves
48
tests/js/pcm-worklet.test.mjs
Normal file
48
tests/js/pcm-worklet.test.mjs
Normal file
@@ -0,0 +1,48 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { test } from "node:test";
|
||||
import vm from "node:vm";
|
||||
|
||||
function processor() {
|
||||
let Processor;
|
||||
class Base {
|
||||
constructor() {
|
||||
this.port = { sent: [], postMessage(data) { this.sent.push(data); } };
|
||||
}
|
||||
}
|
||||
vm.runInNewContext(readFileSync(new URL("../../src/voice_assistant/audio_gateway/web/pcm-worklet.js", import.meta.url), "utf8"), {
|
||||
AudioWorkletProcessor: Base, sampleRate: 16000,
|
||||
registerProcessor(name, constructor) { assert.equal(name, "pcm-input"); Processor = constructor; },
|
||||
});
|
||||
return new Processor();
|
||||
}
|
||||
|
||||
test("negotiated capture emits complete little-endian PCM16 frames", () => {
|
||||
const input = processor();
|
||||
input.port.onmessage({ data: { type: "start", chunkSamples: 2 } });
|
||||
input.process([[new Float32Array([-1, 0, 1])]]);
|
||||
const chunk = input.port.sent[0];
|
||||
assert.equal(chunk.type, "pcm");
|
||||
assert.equal(chunk.buffer.byteLength, 4);
|
||||
const samples = new DataView(chunk.buffer);
|
||||
assert.equal(samples.getInt16(0, true), -32768);
|
||||
assert.equal(samples.getInt16(2, true), 0);
|
||||
});
|
||||
|
||||
test("capture continues across process callbacks until stop", () => {
|
||||
const input = processor();
|
||||
input.port.onmessage({ data: { type: "start", chunkSamples: 2 } });
|
||||
input.process([[new Float32Array([0, 0, 0])]]);
|
||||
input.process([[new Float32Array([0, 0])]]);
|
||||
assert.equal(input.port.sent.filter((event) => event.type === "pcm").length, 2);
|
||||
input.port.onmessage({ data: { type: "stop" } });
|
||||
input.process([[new Float32Array([0, 0, 0])]]);
|
||||
assert.equal(input.port.sent.length, 2);
|
||||
});
|
||||
|
||||
test("invalid capture window reports an error", () => {
|
||||
const input = processor();
|
||||
input.port.onmessage({ data: { type: "start", chunkSamples: 0 } });
|
||||
assert.equal(input.port.sent[0].type, "error");
|
||||
assert.equal(input.capturing, false);
|
||||
});
|
||||
@@ -1,83 +0,0 @@
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { VoiceActivityDetector } from "../../src/voice_assistant/audio_gateway/web/vad.js";
|
||||
|
||||
test("detects speech start after minimum duration", () => {
|
||||
let starts = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 250,
|
||||
onSpeechStart: () => starts++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
|
||||
vad.process(0.04, 0);
|
||||
vad.process(0.04, 100);
|
||||
|
||||
assert.equal(starts, 0);
|
||||
|
||||
vad.process(0.04, 260);
|
||||
|
||||
assert.equal(starts, 1);
|
||||
});
|
||||
|
||||
test("detects speech end after silence", () => {
|
||||
let ends = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 100,
|
||||
silenceMs: 800,
|
||||
onSpeechEnd: () => ends++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
|
||||
vad.process(0.04, 0);
|
||||
vad.process(0.04, 120);
|
||||
vad.process(0.04, 300);
|
||||
|
||||
vad.process(0.001, 600);
|
||||
vad.process(0.001, 1000);
|
||||
|
||||
assert.equal(ends, 0);
|
||||
|
||||
vad.process(0.001, 1101);
|
||||
|
||||
assert.equal(ends, 1);
|
||||
});
|
||||
|
||||
test("ignores short noise spikes", () => {
|
||||
let starts = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 250,
|
||||
onSpeechStart: () => starts++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
|
||||
vad.process(0.05, 0);
|
||||
vad.process(0.001, 80);
|
||||
vad.process(0.001, 300);
|
||||
|
||||
assert.equal(starts, 0);
|
||||
});
|
||||
|
||||
test("stop disables detection", () => {
|
||||
let starts = 0;
|
||||
|
||||
const vad = new VoiceActivityDetector({
|
||||
minSpeechMs: 100,
|
||||
onSpeechStart: () => starts++,
|
||||
});
|
||||
|
||||
vad.start();
|
||||
vad.stop();
|
||||
|
||||
vad.process(0.05, 0);
|
||||
vad.process(0.05, 200);
|
||||
|
||||
assert.equal(starts, 0);
|
||||
});
|
||||
@@ -1,746 +1,270 @@
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { afterEach, test } from "node:test";
|
||||
import { VoiceSessionController } from "../../src/voice_assistant/audio_gateway/web/voice-session.js";
|
||||
|
||||
import { VoiceSessionController } from
|
||||
"../../src/voice_assistant/audio_gateway/web/voice-session.js";
|
||||
|
||||
const originalWebSocket = globalThis.WebSocket;
|
||||
|
||||
class FakeWebSocket {
|
||||
const controllers = [];
|
||||
class Node {
|
||||
constructor() { this.disconnected = false; }
|
||||
connect() {}
|
||||
disconnect() { this.disconnected = true; }
|
||||
}
|
||||
class Context {
|
||||
constructor() {
|
||||
this.destination = {};
|
||||
this.currentTime = 0;
|
||||
this.sources = [];
|
||||
this.audioWorklet = { addModule: async () => {} };
|
||||
this.nextResume = null;
|
||||
}
|
||||
createAnalyser() { return new Node(); }
|
||||
createGain() { return Object.assign(new Node(), { gain: { value: 1 } }); }
|
||||
createMediaStreamSource() { this.input = new Node(); return this.input; }
|
||||
async resume() { const pending = this.nextResume; this.nextResume = null; if (pending) await pending; }
|
||||
createBuffer(channels, length, sampleRate) {
|
||||
return { duration: length / sampleRate, copyToChannel(samples) { this.samples = samples; } };
|
||||
}
|
||||
createBufferSource() {
|
||||
const source = Object.assign(new Node(), {
|
||||
stopped: false, onended: null,
|
||||
start: () => this.sources.push(source),
|
||||
stop() { this.stopped = true; },
|
||||
end() { this.onended?.(); },
|
||||
});
|
||||
return source;
|
||||
}
|
||||
}
|
||||
class Worklet extends Node {
|
||||
constructor() {
|
||||
super();
|
||||
this.port = { messages: [], postMessage(data) { this.messages.push(data); }, onmessage: null };
|
||||
}
|
||||
}
|
||||
class Socket {
|
||||
static OPEN = 1;
|
||||
static CLOSED = 3;
|
||||
|
||||
constructor() {
|
||||
this.readyState = FakeWebSocket.OPEN;
|
||||
this.sent = [];
|
||||
this.closed = false;
|
||||
}
|
||||
|
||||
send(data) {
|
||||
this.sent.push(data);
|
||||
}
|
||||
|
||||
close() {
|
||||
this.closed = true;
|
||||
this.readyState = FakeWebSocket.CLOSED;
|
||||
}
|
||||
constructor(url) { this.url = url; this.readyState = 0; this.sent = []; this.bufferedAmount = 0; }
|
||||
send(data) { if (this.sendFailure) throw new Error("Transport failed"); this.sent.push(data); }
|
||||
open() { this.readyState = 1; this.onopen?.(); }
|
||||
close() { this.readyState = 3; this.onclose?.(); }
|
||||
receive(event) { this.onmessage?.({ data: JSON.stringify(event) }); }
|
||||
}
|
||||
|
||||
globalThis.WebSocket = FakeWebSocket;
|
||||
globalThis.cancelAnimationFrame = () => {};
|
||||
globalThis.requestAnimationFrame = () => 1;
|
||||
|
||||
|
||||
afterEach(() => {
|
||||
// Restore only state modified by individual tests.
|
||||
globalThis.WebSocket = Socket;
|
||||
globalThis.AudioWorkletNode = Worklet;
|
||||
globalThis.location = { protocol: "http:", host: "localhost:8001" };
|
||||
afterEach(() => { for (const controller of controllers.splice(0)) controller.close(); });
|
||||
const tick = () => new Promise((resolve) => setImmediate(resolve));
|
||||
const ready = (overrides = {}) => ({
|
||||
type: "session.ready", turn_detection: "server",
|
||||
audio: { encoding: "pcm_s16le", sample_rate: 16000, channels: 1 },
|
||||
output_audio: { encoding: "pcm_s16le", sample_rate: 48000, channels: 1 },
|
||||
max_audio_chunk_bytes: 65536, ...overrides,
|
||||
});
|
||||
|
||||
function makeController() {
|
||||
const states = [];
|
||||
const errors = [];
|
||||
const workletMessages = [];
|
||||
|
||||
function messages(socket) { return socket.sent.filter((data) => typeof data === "string").map(JSON.parse); }
|
||||
async function setup(options = {}, confirm = true) {
|
||||
const errors = [], states = [], transcripts = [], replies = [];
|
||||
const controller = new VoiceSessionController({
|
||||
onStateChange: (state) => states.push(state),
|
||||
onError: (error) => errors.push(error),
|
||||
onError: (error) => errors.push(error), onStateChange: (state) => states.push(state),
|
||||
onTranscript: (...args) => transcripts.push(args), onAgentText: (...args) => replies.push(args),
|
||||
...options,
|
||||
});
|
||||
|
||||
controller.ws = new FakeWebSocket();
|
||||
controller.worklet = {
|
||||
port: {
|
||||
postMessage: (data) => workletMessages.push(data),
|
||||
onmessage: null,
|
||||
},
|
||||
disconnect() {},
|
||||
};
|
||||
|
||||
controller.source = { disconnect() {} };
|
||||
controller.silentGain = { disconnect() {} };
|
||||
|
||||
return {
|
||||
controller,
|
||||
states,
|
||||
errors,
|
||||
workletMessages,
|
||||
};
|
||||
controllers.push(controller);
|
||||
const context = new Context();
|
||||
await controller.start({ stream: {}, audioContext: context });
|
||||
const socket = controller.ws;
|
||||
socket.open();
|
||||
if (confirm) socket.receive(ready());
|
||||
return { controller, context, socket, errors, states, transcripts, replies };
|
||||
}
|
||||
|
||||
test("speech detection requests a new turn", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.setState("listening");
|
||||
controller.beginTurn();
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[0]),
|
||||
{ type: "audio.start" }
|
||||
);
|
||||
});
|
||||
|
||||
test("PCM is buffered until turn.started", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.setState("listening");
|
||||
|
||||
test("requests server VAD and starts capture only after session.ready", async () => {
|
||||
const { controller, socket } = await setup({}, false);
|
||||
assert.equal(messages(socket)[0].turn_detection, "server");
|
||||
assert.deepEqual(controller.worklet.port.messages, []);
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
assert.equal(socket.sent.length, 1);
|
||||
socket.receive(ready());
|
||||
assert.deepEqual(controller.worklet.port.messages, [{ type: "start", chunkSamples: 1600 }]);
|
||||
assert.equal(controller.state, "listening");
|
||||
});
|
||||
|
||||
assert.equal(controller.prebuffer.length, 1);
|
||||
assert.equal(controller.ws.sent.length, 0);
|
||||
|
||||
controller.beginTurn();
|
||||
controller.handlePCM(new ArrayBuffer(6));
|
||||
|
||||
assert.equal(controller.pending.length, 2);
|
||||
assert.equal(controller.ws.sent.length, 1);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 1,
|
||||
for (const state of ["listening", "recording", "processing", "speaking", "waiting-playback"]) {
|
||||
test(`sends continuous PCM while ${state}, without turn metadata`, async () => {
|
||||
const { controller, socket } = await setup();
|
||||
controller.setState(state);
|
||||
const audio = new ArrayBuffer(4);
|
||||
controller.handlePCM(audio);
|
||||
assert.deepEqual(messages(socket).at(-1), { type: "audio.input", sequence_number: 0 });
|
||||
assert.equal(socket.sent.at(-1), audio);
|
||||
});
|
||||
}
|
||||
|
||||
test("global input sequence never resets when backend opens another turn", async () => {
|
||||
const { controller, socket } = await setup();
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
socket.receive({ type: "interrupted", turn_id: 1 });
|
||||
socket.receive({ type: "turn.started", turn_id: 2 });
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
assert.deepEqual(messages(socket).filter((m) => m.type === "audio.input").map((m) => m.sequence_number), [0, 1, 2]);
|
||||
assert.equal(messages(socket).some((m) => ["audio.start", "audio.stop", "interrupt"].includes(m.type)), false);
|
||||
assert.equal(controller.worklet.port.messages.length, 1);
|
||||
});
|
||||
|
||||
test("uses the negotiated chunk size for microphone capture", async () => {
|
||||
const { controller, socket } = await setup({}, false);
|
||||
socket.receive(ready({ max_audio_chunk_bytes: 1000 }));
|
||||
assert.deepEqual(controller.worklet.port.messages, [{ type: "start", chunkSamples: 500 }]);
|
||||
});
|
||||
|
||||
test("turn.ended waits for speakers without pausing microphone input", async () => {
|
||||
const { controller, context, socket } = await setup();
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
await controller.playbackPending;
|
||||
assert.equal(controller.state, "speaking");
|
||||
socket.receive({ type: "turn.ended", turn_id: 1 });
|
||||
await tick();
|
||||
assert.equal(controller.state, "waiting-playback");
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
assert.equal(messages(socket).at(-1).type, "audio.input");
|
||||
context.sources[0].end();
|
||||
await tick();
|
||||
assert.equal(controller.state, "listening");
|
||||
assert.equal(controller.worklet.port.messages.length, 1);
|
||||
});
|
||||
|
||||
test("a new backend turn stops old playback and ignores its pending completion", async () => {
|
||||
const { controller, context, socket } = await setup();
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
await controller.playbackPending;
|
||||
socket.receive({ type: "turn.ended", turn_id: 1 });
|
||||
await tick();
|
||||
socket.receive({ type: "turn.started", turn_id: 2 });
|
||||
await tick();
|
||||
assert.equal(context.sources[0].stopped, true);
|
||||
assert.equal(controller.turnId, 2);
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.pending.length, 0);
|
||||
assert.equal(controller.sequence, 2);
|
||||
assert.equal(controller.ws.sent.length, 5);
|
||||
});
|
||||
|
||||
test("PCM metadata precedes each binary frame", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.turnId = 7;
|
||||
controller.setState("recording");
|
||||
|
||||
const pcm = new ArrayBuffer(8);
|
||||
|
||||
controller.sendPCM(pcm);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[0]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 7,
|
||||
sequence_number: 0,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(controller.ws.sent[1], pcm);
|
||||
assert.equal(controller.sequence, 1);
|
||||
test("old queued PCM is discarded after playback generation changes", async () => {
|
||||
const { controller, context, socket } = await setup();
|
||||
let release;
|
||||
context.nextResume = new Promise((resolve) => { release = resolve; });
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
const obsolete = controller.playbackPending;
|
||||
await tick();
|
||||
socket.receive({ type: "turn.started", turn_id: 2 });
|
||||
release();
|
||||
await obsolete;
|
||||
assert.equal(context.sources.length, 0);
|
||||
controller.handleAudioChunk(new ArrayBuffer(4));
|
||||
await controller.playbackPending;
|
||||
assert.equal(context.sources.length, 1);
|
||||
});
|
||||
|
||||
test("speech end requests worklet flush", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("recording");
|
||||
|
||||
controller.finishSpeech();
|
||||
|
||||
assert.equal(controller.state, "flushing");
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
|
||||
assert.equal(controller.ws.sent.length, 0);
|
||||
test("a failure in obsolete playback does not close the new turn", async () => {
|
||||
const { controller, context, socket, errors } = await setup();
|
||||
let reject;
|
||||
context.nextResume = new Promise((resolve, fail) => { reject = fail; });
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
const obsolete = controller.playbackPending;
|
||||
await tick();
|
||||
socket.receive({ type: "turn.started", turn_id: 2 });
|
||||
reject(new Error("Old playback failed"));
|
||||
await obsolete;
|
||||
assert.equal(errors.length, 0);
|
||||
assert.equal(controller.turnId, 2);
|
||||
assert.equal(socket.readyState, Socket.OPEN);
|
||||
});
|
||||
|
||||
|
||||
test("audio.stop is sent after flushed", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("flushing");
|
||||
|
||||
controller.handleFlushed();
|
||||
|
||||
const messages = controller.ws.sent
|
||||
.filter((message) => typeof message === "string")
|
||||
.map((message) => JSON.parse(message));
|
||||
|
||||
assert.ok(
|
||||
messages.some(
|
||||
(message) =>
|
||||
message.type === "audio.stop" &&
|
||||
message.turn_id === 1
|
||||
)
|
||||
);
|
||||
|
||||
assert.equal(controller.state, "processing");
|
||||
test("stale turn events cannot change the current response", async () => {
|
||||
const { controller, socket, transcripts, replies } = await setup();
|
||||
socket.receive({ type: "turn.started", turn_id: 2 });
|
||||
socket.receive({ type: "stt.final", turn_id: 1, text: "old" });
|
||||
socket.receive({ type: "agent.response", turn_id: 1, text: "old" });
|
||||
socket.receive({ type: "interrupted", turn_id: 1 });
|
||||
socket.receive({ type: "turn.ended", turn_id: 1 });
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
assert.equal(controller.turnId, 2);
|
||||
assert.deepEqual(transcripts, []);
|
||||
assert.deepEqual(replies, []);
|
||||
socket.receive({ type: "stt.final", turn_id: 2, text: "new" });
|
||||
socket.receive({ type: "agent.response", turn_id: 2, text: "reply" });
|
||||
assert.deepEqual(transcripts, [["new", true]]);
|
||||
assert.deepEqual(replies, [["reply", true]]);
|
||||
});
|
||||
|
||||
test("interrupted returns to listening without stopping capture or starting a client turn", async () => {
|
||||
const { controller, socket } = await setup();
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
socket.receive({ type: "interrupted", turn_id: 1 });
|
||||
assert.equal(controller.state, "listening");
|
||||
assert.equal(controller.turnId, null);
|
||||
assert.deepEqual(messages(socket).map((m) => m.type), ["session.start"]);
|
||||
assert.equal(controller.worklet.port.messages.length, 1);
|
||||
});
|
||||
|
||||
test("speech ending before turn.started is handled", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.setState("listening");
|
||||
controller.beginTurn();
|
||||
controller.finishSpeech();
|
||||
|
||||
assert.equal(controller.stopRequested, true);
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 3,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "flushing");
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
});
|
||||
|
||||
test("closing a session releases resources", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
const socket = controller.ws;
|
||||
|
||||
controller.setState("listening");
|
||||
controller.close();
|
||||
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(socket.closed, true);
|
||||
assert.equal(controller.ws, null);
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "stop" }
|
||||
);
|
||||
});
|
||||
|
||||
test("gateway errors close the session", () => {
|
||||
const { controller, errors } = makeController();
|
||||
|
||||
controller.setState("processing");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "error",
|
||||
code: "STT_START_FAILED",
|
||||
message: "Provider unavailable",
|
||||
for (const metadata of [{ turn_detection: "client" }, { turn_detection: undefined }, { max_audio_chunk_bytes: 1 }]) {
|
||||
test(`rejects incompatible session metadata: ${JSON.stringify(metadata)}`, async () => {
|
||||
const { controller, socket, errors } = await setup({}, false);
|
||||
socket.receive(ready(metadata));
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(errors.length, 1);
|
||||
});
|
||||
}
|
||||
|
||||
test("backpressure closes the session before queuing more PCM", async () => {
|
||||
const { controller, socket, errors } = await setup({ maxBufferedBytes: 8 });
|
||||
socket.bufferedAmount = 8;
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(errors.length, 1);
|
||||
assert.match(
|
||||
errors[0].message,
|
||||
/STT_START_FAILED/
|
||||
);
|
||||
assert.equal(messages(socket).some((m) => m.type === "audio.input"), false);
|
||||
});
|
||||
|
||||
|
||||
test("turn.ended waits for playback before listening", async () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let resolvePlayback;
|
||||
|
||||
controller.player = {
|
||||
finish() {},
|
||||
reset() {},
|
||||
stop() {},
|
||||
waitForCompletion() {
|
||||
return new Promise((resolve) => {
|
||||
resolvePlayback = resolve;
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("processing");
|
||||
|
||||
let resumed = false;
|
||||
|
||||
controller.startListening = () => {
|
||||
resumed = true;
|
||||
};
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.ended",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(resumed, false);
|
||||
|
||||
resolvePlayback();
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(resumed, true);
|
||||
test("invalid PCM closes the session without emitting metadata", async () => {
|
||||
const { controller, socket, errors } = await setup();
|
||||
controller.handlePCM(new ArrayBuffer(3));
|
||||
assert.equal(errors.length, 1);
|
||||
assert.equal(messages(socket).some((m) => m.type === "audio.input"), false);
|
||||
});
|
||||
|
||||
test("closing during playback prevents listening resume", async () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let resolvePlayback;
|
||||
|
||||
controller.player = {
|
||||
finish() {},
|
||||
reset() {},
|
||||
stop() {},
|
||||
waitForCompletion() {
|
||||
return new Promise((resolve) => {
|
||||
resolvePlayback = resolve;
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("processing");
|
||||
|
||||
let resumed = false;
|
||||
|
||||
controller.startListening = () => {
|
||||
resumed = true;
|
||||
};
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.ended",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
controller.close();
|
||||
|
||||
resolvePlayback();
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
assert.equal(resumed, false);
|
||||
assert.equal(controller.state, "closed");
|
||||
});
|
||||
|
||||
test("binary audio is forwarded to PCMPlayer", async () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
const received = [];
|
||||
|
||||
controller.player = {
|
||||
async enqueue(buffer) {
|
||||
received.push(buffer);
|
||||
},
|
||||
finish() {},
|
||||
reset() {},
|
||||
stop() {},
|
||||
waitForCompletion: async () => {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("processing");
|
||||
|
||||
const pcm = new ArrayBuffer(8);
|
||||
|
||||
controller.handleAudioChunk(pcm);
|
||||
|
||||
await controller.playbackPending;
|
||||
|
||||
assert.deepEqual(received, [pcm]);
|
||||
});
|
||||
|
||||
|
||||
|
||||
test("barge-in stops playback and starts a new turn", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let playbackStopped = false;
|
||||
|
||||
controller.player = {
|
||||
stop() {
|
||||
playbackStopped = true;
|
||||
},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
// Simulate user speech detected during TTS.
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(playbackStopped, true);
|
||||
assert.equal(controller.state, "interrupting");
|
||||
assert.equal(controller.bargeInPending, true);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[0]),
|
||||
{
|
||||
type: "interrupt",
|
||||
turn_id: 1,
|
||||
}
|
||||
);
|
||||
|
||||
// Gateway confirms the previous turn was cancelled.
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
assert.equal(controller.ws.closed, false);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[1]),
|
||||
{
|
||||
type: "audio.start",
|
||||
}
|
||||
);
|
||||
|
||||
// Gateway accepts the new turn.
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.turnId, 2);
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.bargeInPending, false);
|
||||
});
|
||||
|
||||
|
||||
|
||||
test("barge-in preserves PCM until new turn starts", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
const pcm1 = new ArrayBuffer(4);
|
||||
const pcm2 = new ArrayBuffer(6);
|
||||
const pcm3 = new ArrayBuffer(8);
|
||||
|
||||
// Audio captured before speech confirmation.
|
||||
controller.handlePCM(pcm1);
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
// Audio captured while interruption is pending.
|
||||
controller.handlePCM(pcm2);
|
||||
|
||||
assert.equal(controller.ws.sent.length, 1);
|
||||
assert.equal(controller.bargeInBuffer.length, 2);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
// Still no PCM sent to the old turn.
|
||||
assert.equal(controller.state, "starting");
|
||||
assert.equal(controller.ws.sent.length, 2);
|
||||
|
||||
// Additional speech arrives before turn.started.
|
||||
controller.handlePCM(pcm3);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
const sent = controller.ws.sent;
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[0]),
|
||||
{ type: "interrupt", turn_id: 1 }
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[1]),
|
||||
{ type: "audio.start" }
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[2]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 0,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(sent[3], pcm1);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[4]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 1,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(sent[5], pcm2);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[6]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 2,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(sent[7], pcm3);
|
||||
|
||||
assert.equal(controller.sequence, 3);
|
||||
assert.equal(controller.state, "recording");
|
||||
});
|
||||
|
||||
|
||||
test("barge-in finishes speech before new turn starts", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
const pcm = new ArrayBuffer(8);
|
||||
|
||||
controller.handlePCM(pcm);
|
||||
controller.requestBargeIn();
|
||||
|
||||
// User finishes speaking before the Gateway responds.
|
||||
controller.finishBargeInSpeech();
|
||||
|
||||
assert.equal(controller.bargeInSpeechEnded, true);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
// Buffered audio must be transmitted before flushing.
|
||||
assert.equal(controller.sequence, 1);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[2]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 0,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(controller.ws.sent[3], pcm);
|
||||
assert.equal(controller.state, "flushing");
|
||||
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
test("barge-in continues recording until speech ends", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.bargeInVad.start();
|
||||
controller.requestBargeIn();
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.bargeInRecording, true);
|
||||
assert.equal(controller.bargeInVad.active, true);
|
||||
|
||||
// Speech continues after the new turn starts.
|
||||
const pcm = new ArrayBuffer(8);
|
||||
controller.handlePCM(pcm);
|
||||
|
||||
assert.equal(controller.sequence, 1);
|
||||
|
||||
// Barge-in VAD detects the end of speech.
|
||||
controller.finishBargeInSpeech();
|
||||
|
||||
assert.equal(controller.bargeInRecording, false);
|
||||
assert.equal(controller.state, "flushing");
|
||||
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
|
||||
// Worklet confirms that remaining PCM was emitted.
|
||||
controller.handleFlushed();
|
||||
|
||||
assert.equal(controller.state, "processing");
|
||||
|
||||
const messages = controller.ws.sent
|
||||
.filter((message) => typeof message === "string")
|
||||
.map((message) => JSON.parse(message));
|
||||
|
||||
assert.ok(
|
||||
messages.some(
|
||||
(message) =>
|
||||
message.type === "audio.stop" &&
|
||||
message.turn_id === 2
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("late INVALID_TURN recovers barge-in session", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(controller.state, "interrupting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "error",
|
||||
code: "INVALID_TURN",
|
||||
message: "Turn 1 is no longer active.",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
assert.equal(controller.ws.closed, false);
|
||||
|
||||
const messages = controller.ws.sent
|
||||
.filter((message) => typeof message === "string")
|
||||
.map((message) => JSON.parse(message));
|
||||
|
||||
assert.deepEqual(
|
||||
messages.map((message) => message.type),
|
||||
["interrupt", "audio.start"]
|
||||
);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.turnId, 2);
|
||||
});
|
||||
|
||||
|
||||
test("late TTS chunk during barge-in does not close session", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let stopped = false;
|
||||
|
||||
controller.player = {
|
||||
stop() {
|
||||
stopped = true;
|
||||
},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(stopped, true);
|
||||
assert.equal(controller.state, "interrupting");
|
||||
|
||||
// A late PCM chunk from the interrupted TTS.
|
||||
test("close stops capture and playback even if session.end cannot be sent", async () => {
|
||||
const { controller, socket, context } = await setup();
|
||||
const worklet = controller.worklet;
|
||||
socket.receive({ type: "turn.started", turn_id: 1 });
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
|
||||
// The WebSocket session must remain alive.
|
||||
assert.equal(controller.ws.closed, false);
|
||||
assert.equal(controller.state, "interrupting");
|
||||
assert.equal(controller.bargeInPending, true);
|
||||
await controller.playbackPending;
|
||||
socket.sendFailure = true;
|
||||
controller.close();
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(worklet.disconnected, true);
|
||||
assert.deepEqual(worklet.port.messages.at(-1), { type: "stop" });
|
||||
assert.equal(context.sources[0].stopped, true);
|
||||
assert.equal(socket.readyState, Socket.CLOSED);
|
||||
});
|
||||
|
||||
|
||||
test("barge-in resets player before new TTS response", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let finished = false;
|
||||
let resetCount = 0;
|
||||
|
||||
controller.player = {
|
||||
stop() {
|
||||
finished = true;
|
||||
},
|
||||
|
||||
reset() {
|
||||
finished = false;
|
||||
resetCount++;
|
||||
},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(finished, true);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(finished, false);
|
||||
assert.equal(resetCount, 1);
|
||||
test("closing during worklet loading prevents orphan microphone graphs and sockets", async () => {
|
||||
let release;
|
||||
const context = new Context();
|
||||
context.audioWorklet.addModule = () => new Promise((resolve) => { release = resolve; });
|
||||
const controller = new VoiceSessionController();
|
||||
controllers.push(controller);
|
||||
const starting = controller.start({ stream: {}, audioContext: context });
|
||||
controller.close();
|
||||
release();
|
||||
await starting;
|
||||
assert.equal(controller.ws, null);
|
||||
assert.equal(context.input, undefined);
|
||||
});
|
||||
|
||||
|
||||
test("session.ready configures output before starting microphone capture", () => {
|
||||
const { controller } = makeController();
|
||||
const operations = [];
|
||||
const format = { encoding: "pcm_s16le", sample_rate: 24000, channels: 1 };
|
||||
controller.player = { configure: (received) => operations.push(received) };
|
||||
controller.startListening = () => operations.push("listen");
|
||||
controller.handleGatewayEvent({ type: "session.ready", output_audio: format });
|
||||
assert.deepEqual(operations, [format, "listen"]);
|
||||
test("missing session.ready times out and closes the connection", async () => {
|
||||
const { controller, errors } = await setup({ handshakeTimeoutMs: 5 }, false);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
assert.equal(controller.state, "closed");
|
||||
assert.equal(errors.length, 1);
|
||||
});
|
||||
|
||||
223
tests/test_continuous_gateway.py
Normal file
223
tests/test_continuous_gateway.py
Normal file
@@ -0,0 +1,223 @@
|
||||
"""Continuous PCM WebSocket mode delegates turn decisions to backend VAD."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from voice_assistant.agent.mock import MockAgentAdapter
|
||||
from voice_assistant.audio_gateway.app import main
|
||||
from voice_assistant.stt.mock import MockSTTAdapter
|
||||
from voice_assistant.tts.mock import MockTTSAdapter
|
||||
from voice_assistant.vad.adapter import VADAdapter
|
||||
from voice_assistant.vad.models import SpeechStarted, SpeechEnded
|
||||
|
||||
|
||||
class VAD(VADAdapter):
|
||||
def __init__(self):
|
||||
self.script = []
|
||||
self.tail = ()
|
||||
self.received = []
|
||||
self.closed = False
|
||||
|
||||
async def process(self, audio):
|
||||
self.received.append(audio)
|
||||
return self.script.pop(0) if self.script else ()
|
||||
|
||||
async def flush(self):
|
||||
return self.tail
|
||||
|
||||
async def reset(self):
|
||||
pass
|
||||
|
||||
async def aclose(self):
|
||||
self.closed = True
|
||||
|
||||
|
||||
class STT(MockSTTAdapter):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.received = []
|
||||
|
||||
async def send_audio(self, audio):
|
||||
await super().send_audio(audio)
|
||||
self.received.append(audio)
|
||||
|
||||
async def stop_session(self):
|
||||
await self.emit_final('Pergunta de teste')
|
||||
await super().stop_session()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def components(monkeypatch):
|
||||
detector = VAD()
|
||||
providers = []
|
||||
|
||||
def create_stt(**kwargs):
|
||||
stt = STT()
|
||||
providers.append(stt)
|
||||
return stt
|
||||
|
||||
monkeypatch.setattr(main, 'create_vad_adapter', lambda **kwargs: detector)
|
||||
monkeypatch.setattr(main, 'create_stt_adapter', create_stt)
|
||||
monkeypatch.setattr(main, 'create_agent_adapter', lambda **kwargs: MockAgentAdapter())
|
||||
monkeypatch.setattr(main, 'create_tts_adapter', lambda **kwargs: MockTTSAdapter())
|
||||
return detector, providers
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connect(mode='server'):
|
||||
with TestClient(main.app).websocket_connect('/ws/audio') as socket:
|
||||
socket.send_json({'type': 'session.start', 'turn_detection': mode})
|
||||
ready = socket.receive_json()
|
||||
assert ready['type'] == 'session.ready'
|
||||
yield socket, ready
|
||||
|
||||
|
||||
def send_audio(socket, sequence, audio):
|
||||
socket.send_json({'type': 'audio.input', 'sequence_number': sequence})
|
||||
socket.send_bytes(audio)
|
||||
|
||||
|
||||
def read_turn(socket):
|
||||
events, audio = [], []
|
||||
for _ in range(20):
|
||||
message = socket.receive()
|
||||
if message.get('bytes') is not None:
|
||||
audio.append(message['bytes'])
|
||||
else:
|
||||
import json
|
||||
event = json.loads(message['text'])
|
||||
assert event['type'] != 'error', event
|
||||
events.append(event)
|
||||
if event['type'] == 'turn.ended':
|
||||
return events, audio
|
||||
raise AssertionError('Turn did not complete within expected events')
|
||||
|
||||
|
||||
def test_continuous_mode_receives_audio_before_first_turn(components):
|
||||
detector, providers = components
|
||||
with connect() as (socket, ready):
|
||||
assert ready['turn_detection'] == 'server'
|
||||
assert ready['max_audio_chunk_bytes'] == 65536
|
||||
send_audio(socket, 0, bytes(512))
|
||||
socket.send_json({'type': 'session.end'})
|
||||
assert socket.receive_json()['type'] == 'session.ended'
|
||||
assert detector.received == [bytes(512)]
|
||||
assert providers == []
|
||||
assert detector.closed
|
||||
|
||||
|
||||
def test_vad_opens_and_closes_turn_and_input_sequence_survives_next_turn(components):
|
||||
detector, providers = components
|
||||
detector.script = [
|
||||
(SpeechStarted(0),), (SpeechEnded(400, 'silence'),),
|
||||
(), (SpeechStarted(900),),
|
||||
]
|
||||
detector.tail = (SpeechEnded(1024, 'end_of_input'),)
|
||||
with connect() as (socket, ready):
|
||||
send_audio(socket, 0, bytes(512))
|
||||
assert socket.receive_json()['type'] == 'turn.started'
|
||||
send_audio(socket, 1, bytes(512))
|
||||
events, audio = read_turn(socket)
|
||||
assert events[-1]['turn_id'] == 1
|
||||
assert any(event['type'] == 'stt.final' for event in events)
|
||||
assert b''.join(audio) == b'\x01\x02\x03\x04\x05\x06\x07\x08'
|
||||
send_audio(socket, 2, bytes(512))
|
||||
send_audio(socket, 3, bytes(512))
|
||||
assert socket.receive_json()['turn_id'] == 2
|
||||
socket.send_json({'type': 'audio.input.end'})
|
||||
events, audio = read_turn(socket)
|
||||
assert events[-1]['turn_id'] == 2
|
||||
assert audio
|
||||
assert len(providers) == 2
|
||||
assert len(detector.received) == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize('sequence', [-1, True, 0.5, '0', 1, 2])
|
||||
def test_bad_sequence_is_rejected_without_consuming_input(components, sequence):
|
||||
detector, _ = components
|
||||
with connect() as (socket, ready):
|
||||
socket.send_json({'type': 'audio.input', 'sequence_number': sequence})
|
||||
error = socket.receive_json()
|
||||
assert error['code'] == 'AUDIO_SEQUENCE_ERROR'
|
||||
assert error['details']['expected_sequence_number'] == 0
|
||||
send_audio(socket, 0, bytes(512))
|
||||
socket.send_json({'type': 'session.end'})
|
||||
assert socket.receive_json()['type'] == 'session.ended'
|
||||
assert len(detector.received) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(('audio', 'code'), [
|
||||
(b'', 'AUDIO_SEQUENCE_ERROR'), (bytes(3), 'AUDIO_SEQUENCE_ERROR'),
|
||||
(bytes(6), 'MESSAGE_TOO_LARGE'),
|
||||
])
|
||||
def test_bad_pcm_does_not_advance_global_sequence(components, monkeypatch, audio, code):
|
||||
detector, _ = components
|
||||
config = main.app_config.model_copy(update={'voice_input': {'max_input_chunk_bytes': 4}})
|
||||
monkeypatch.setattr(main, 'app_config', config)
|
||||
with connect() as (socket, ready):
|
||||
assert ready['max_audio_chunk_bytes'] == 4
|
||||
send_audio(socket, 0, audio)
|
||||
assert socket.receive_json()['code'] == code
|
||||
send_audio(socket, 0, bytes(4))
|
||||
socket.send_json({'type': 'session.end'})
|
||||
assert socket.receive_json()['type'] == 'session.ended'
|
||||
assert detector.received == [bytes(4)]
|
||||
|
||||
|
||||
def test_duplicate_metadata_does_not_overwrite_pending_pcm(components):
|
||||
detector, _ = components
|
||||
with connect() as (socket, ready):
|
||||
socket.send_json({'type': 'audio.input', 'sequence_number': 0})
|
||||
socket.send_json({'type': 'audio.input', 'sequence_number': 1})
|
||||
assert socket.receive_json()['code'] == 'AUDIO_SEQUENCE_ERROR'
|
||||
socket.send_bytes(bytes(512))
|
||||
socket.send_json({'type': 'session.end'})
|
||||
assert socket.receive_json()['type'] == 'session.ended'
|
||||
assert len(detector.received) == 1
|
||||
|
||||
|
||||
def test_binary_frame_requires_metadata(components):
|
||||
with connect() as (socket, ready):
|
||||
socket.send_bytes(bytes(512))
|
||||
assert socket.receive_json()['code'] == 'INVALID_AUDIO_FRAME'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('event', [
|
||||
{'type': 'audio.start'}, {'type': 'audio.stop', 'turn_id': 1},
|
||||
{'type': 'audio.chunk', 'turn_id': 1, 'sequence_number': 0},
|
||||
])
|
||||
def test_client_turn_commands_are_rejected_in_server_mode(components, event):
|
||||
with connect() as (socket, ready):
|
||||
socket.send_json(event)
|
||||
assert socket.receive_json()['code'] == 'TURN_CONTROLLED_BY_VAD'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('event', [
|
||||
{'type': 'audio.input', 'sequence_number': 0}, {'type': 'audio.input.end'},
|
||||
])
|
||||
def test_manual_mode_rejects_continuous_input(components, event):
|
||||
with connect('client') as (socket, ready):
|
||||
assert ready['turn_detection'] == 'client'
|
||||
socket.send_json(event)
|
||||
assert socket.receive_json()['code'] == 'INVALID_INPUT_MODE'
|
||||
|
||||
|
||||
def test_input_end_cannot_overtake_pending_binary_frame(components):
|
||||
with connect() as (socket, ready):
|
||||
socket.send_json({'type': 'audio.input', 'sequence_number': 0})
|
||||
socket.send_json({'type': 'audio.input.end'})
|
||||
assert socket.receive_json()['code'] == 'INVALID_MESSAGE'
|
||||
|
||||
|
||||
def test_initialization_failure_is_reported_and_can_be_followed_by_manual_session(components, monkeypatch):
|
||||
def unavailable(**kwargs):
|
||||
raise ValueError('VAD unavailable')
|
||||
|
||||
monkeypatch.setattr(main, 'create_vad_adapter', unavailable)
|
||||
with TestClient(main.app).websocket_connect('/ws/audio') as socket:
|
||||
socket.send_json({'type': 'session.start', 'turn_detection': 'server'})
|
||||
assert socket.receive_json()['code'] == 'SESSION_START_FAILED'
|
||||
socket.send_json({'type': 'session.start'})
|
||||
assert socket.receive_json()['type'] == 'session.ready'
|
||||
@@ -95,6 +95,8 @@ def test_session_ready_output():
|
||||
"sample_rate": 16000,
|
||||
"channels": 1,
|
||||
},
|
||||
"turn_detection": "client",
|
||||
"max_audio_chunk_bytes": 65536,
|
||||
"output_audio": {
|
||||
"encoding": "pcm_s16le",
|
||||
"sample_rate": 48000,
|
||||
|
||||
@@ -32,3 +32,13 @@ def test_web_client_audio_worklet():
|
||||
assert response.status_code == 200
|
||||
assert "registerProcessor" in response.text
|
||||
assert "pcm-input" in response.text
|
||||
|
||||
|
||||
|
||||
def test_active_web_client_uses_continuous_server_detection():
|
||||
response = client.get("/web/voice-session.js")
|
||||
assert response.status_code == 200
|
||||
assert 'turn_detection: "server"' in response.text
|
||||
assert 'type: "audio.input"' in response.text
|
||||
assert "VoiceActivityDetector" not in response.text
|
||||
assert client.get("/web/vad.js").status_code == 404
|
||||
|
||||
Reference in New Issue
Block a user