primeira versão customizada de VAD- Voice Activity Detection

2026-10-30 19:15:01 +00:00
parent 2449b34137
commit aba05cb826
14 changed files with 3457 additions and 129 deletions
--- a/tests/js/voice-session.test.mjs
+++ b/tests/js/voice-session.test.mjs
@@ -29,6 +29,8 @@ class FakeWebSocket {

 globalThis.WebSocket = FakeWebSocket;
 globalThis.cancelAnimationFrame = () => {};
+globalThis.requestAnimationFrame = () => 1;
+

 afterEach(() => {
  // Restore only state modified by individual tests.
@@ -145,27 +147,31 @@ test("speech end requests worklet flush", () => {
  assert.equal(controller.ws.sent.length, 0);
 });

+
 test("audio.stop is sent after flushed", () => {
  const { controller } = makeController();

  controller.turnId = 1;
  controller.setState("flushing");

-  controller.handlePCM(new ArrayBuffer(4));
  controller.handleFlushed();

+  const messages = controller.ws.sent
+    .filter((message) => typeof message === "string")
+    .map((message) => JSON.parse(message));
+
+  assert.ok(
+    messages.some(
+      (message) =>
+        message.type === "audio.stop" &&
+        message.turn_id === 1
+    )
+  );
+
  assert.equal(controller.state, "processing");
-
-  const lastMessage = JSON.parse(controller.ws.sent.at(-1));
-
-  assert.deepEqual(lastMessage, {
-    type: "audio.stop",
-    turn_id: 1,
-  });
-
-  assert.equal(controller.ws.sent.length, 3);
 });

+
 test("speech ending before turn.started is handled", () => {
  const { controller, workletMessages } = makeController();

@@ -337,3 +343,393 @@ test("binary audio is forwarded to PCMPlayer", async () => {

  assert.deepEqual(received, [pcm]);
 });
+
+
+
+test("barge-in stops playback and starts a new turn", () => {
+  const { controller } = makeController();
+
+  let playbackStopped = false;
+
+  controller.player = {
+    stop() {
+      playbackStopped = true;
+    },
+    reset() {},
+  };
+
+  controller.turnId = 1;
+  controller.setState("speaking");
+
+  // Simulate user speech detected during TTS.
+  controller.requestBargeIn();
+
+  assert.equal(playbackStopped, true);
+  assert.equal(controller.state, "interrupting");
+  assert.equal(controller.bargeInPending, true);
+
+  assert.deepEqual(
+    JSON.parse(controller.ws.sent[0]),
+    {
+      type: "interrupt",
+      turn_id: 1,
+    }
+  );
+
+  // Gateway confirms the previous turn was cancelled.
+  controller.handleGatewayEvent({
+    type: "interrupted",
+    turn_id: 1,
+  });
+
+  assert.equal(controller.state, "starting");
+  assert.equal(controller.ws.closed, false);
+
+  assert.deepEqual(
+    JSON.parse(controller.ws.sent[1]),
+    {
+      type: "audio.start",
+    }
+  );
+
+  // Gateway accepts the new turn.
+  controller.handleGatewayEvent({
+    type: "turn.started",
+    turn_id: 2,
+  });
+
+  assert.equal(controller.turnId, 2);
+  assert.equal(controller.state, "recording");
+  assert.equal(controller.bargeInPending, false);
+});
+
+
+
+test("barge-in preserves PCM until new turn starts", () => {
+  const { controller } = makeController();
+
+  controller.player = {
+    stop() {},
+    reset() {},
+  };
+
+  controller.turnId = 1;
+  controller.setState("speaking");
+
+  const pcm1 = new ArrayBuffer(4);
+  const pcm2 = new ArrayBuffer(6);
+  const pcm3 = new ArrayBuffer(8);
+
+  // Audio captured before speech confirmation.
+  controller.handlePCM(pcm1);
+
+  controller.requestBargeIn();
+
+  // Audio captured while interruption is pending.
+  controller.handlePCM(pcm2);
+
+  assert.equal(controller.ws.sent.length, 1);
+  assert.equal(controller.bargeInBuffer.length, 2);
+
+  controller.handleGatewayEvent({
+    type: "interrupted",
+    turn_id: 1,
+  });
+
+  // Still no PCM sent to the old turn.
+  assert.equal(controller.state, "starting");
+  assert.equal(controller.ws.sent.length, 2);
+
+  // Additional speech arrives before turn.started.
+  controller.handlePCM(pcm3);
+
+  controller.handleGatewayEvent({
+    type: "turn.started",
+    turn_id: 2,
+  });
+
+  const sent = controller.ws.sent;
+
+  assert.deepEqual(
+    JSON.parse(sent[0]),
+    { type: "interrupt", turn_id: 1 }
+  );
+
+  assert.deepEqual(
+    JSON.parse(sent[1]),
+    { type: "audio.start" }
+  );
+
+  assert.deepEqual(
+    JSON.parse(sent[2]),
+    {
+      type: "audio.chunk",
+      turn_id: 2,
+      sequence_number: 0,
+    }
+  );
+
+  assert.equal(sent[3], pcm1);
+
+  assert.deepEqual(
+    JSON.parse(sent[4]),
+    {
+      type: "audio.chunk",
+      turn_id: 2,
+      sequence_number: 1,
+    }
+  );
+
+  assert.equal(sent[5], pcm2);
+
+  assert.deepEqual(
+    JSON.parse(sent[6]),
+    {
+      type: "audio.chunk",
+      turn_id: 2,
+      sequence_number: 2,
+    }
+  );
+
+  assert.equal(sent[7], pcm3);
+
+  assert.equal(controller.sequence, 3);
+  assert.equal(controller.state, "recording");
+});
+
+
+test("barge-in finishes speech before new turn starts", () => {
+  const { controller, workletMessages } = makeController();
+
+  controller.player = {
+    stop() {},
+    reset() {},
+  };
+
+  controller.turnId = 1;
+  controller.setState("speaking");
+
+  const pcm = new ArrayBuffer(8);
+
+  controller.handlePCM(pcm);
+  controller.requestBargeIn();
+
+  // User finishes speaking before the Gateway responds.
+  controller.finishBargeInSpeech();
+
+  assert.equal(controller.bargeInSpeechEnded, true);
+
+  controller.handleGatewayEvent({
+    type: "interrupted",
+    turn_id: 1,
+  });
+
+  assert.equal(controller.state, "starting");
+
+  controller.handleGatewayEvent({
+    type: "turn.started",
+    turn_id: 2,
+  });
+
+  // Buffered audio must be transmitted before flushing.
+  assert.equal(controller.sequence, 1);
+
+  assert.deepEqual(
+    JSON.parse(controller.ws.sent[2]),
+    {
+      type: "audio.chunk",
+      turn_id: 2,
+      sequence_number: 0,
+    }
+  );
+
+  assert.equal(controller.ws.sent[3], pcm);
+  assert.equal(controller.state, "flushing");
+
+  assert.deepEqual(
+    workletMessages.at(-1),
+    { type: "flush" }
+  );
+});
+
+
+
+
+test("barge-in continues recording until speech ends", () => {
+  const { controller, workletMessages } = makeController();
+
+  controller.player = {
+    stop() {},
+    reset() {},
+  };
+
+  controller.turnId = 1;
+  controller.setState("speaking");
+
+  controller.bargeInVad.start();
+  controller.requestBargeIn();
+
+  controller.handleGatewayEvent({
+    type: "interrupted",
+    turn_id: 1,
+  });
+
+  controller.handleGatewayEvent({
+    type: "turn.started",
+    turn_id: 2,
+  });
+
+  assert.equal(controller.state, "recording");
+  assert.equal(controller.bargeInRecording, true);
+  assert.equal(controller.bargeInVad.active, true);
+
+  // Speech continues after the new turn starts.
+  const pcm = new ArrayBuffer(8);
+  controller.handlePCM(pcm);
+
+  assert.equal(controller.sequence, 1);
+
+  // Barge-in VAD detects the end of speech.
+  controller.finishBargeInSpeech();
+
+  assert.equal(controller.bargeInRecording, false);
+  assert.equal(controller.state, "flushing");
+
+  assert.deepEqual(
+    workletMessages.at(-1),
+    { type: "flush" }
+  );
+
+  // Worklet confirms that remaining PCM was emitted.
+  controller.handleFlushed();
+
+  assert.equal(controller.state, "processing");
+
+  const messages = controller.ws.sent
+    .filter((message) => typeof message === "string")
+    .map((message) => JSON.parse(message));
+
+  assert.ok(
+    messages.some(
+      (message) =>
+        message.type === "audio.stop" &&
+        message.turn_id === 2
+    )
+  );
+});
+
+
+test("late INVALID_TURN recovers barge-in session", () => {
+  const { controller } = makeController();
+
+  controller.player = {
+    stop() {},
+    reset() {},
+  };
+
+  controller.turnId = 1;
+  controller.setState("speaking");
+
+  controller.requestBargeIn();
+
+  assert.equal(controller.state, "interrupting");
+
+  controller.handleGatewayEvent({
+    type: "error",
+    code: "INVALID_TURN",
+    message: "Turn 1 is no longer active.",
+    turn_id: 1,
+  });
+
+  assert.equal(controller.state, "starting");
+  assert.equal(controller.ws.closed, false);
+
+  const messages = controller.ws.sent
+    .filter((message) => typeof message === "string")
+    .map((message) => JSON.parse(message));
+
+  assert.deepEqual(
+    messages.map((message) => message.type),
+    ["interrupt", "audio.start"]
+  );
+
+  controller.handleGatewayEvent({
+    type: "turn.started",
+    turn_id: 2,
+  });
+
+  assert.equal(controller.state, "recording");
+  assert.equal(controller.turnId, 2);
+});
+
+
+test("late TTS chunk during barge-in does not close session", () => {
+  const { controller } = makeController();
+
+  let stopped = false;
+
+  controller.player = {
+    stop() {
+      stopped = true;
+    },
+    reset() {},
+  };
+
+  controller.turnId = 1;
+  controller.setState("speaking");
+
+  controller.requestBargeIn();
+
+  assert.equal(stopped, true);
+  assert.equal(controller.state, "interrupting");
+
+  // A late PCM chunk from the interrupted TTS.
+  controller.handleAudioChunk(new ArrayBuffer(8));
+
+  // The WebSocket session must remain alive.
+  assert.equal(controller.ws.closed, false);
+  assert.equal(controller.state, "interrupting");
+  assert.equal(controller.bargeInPending, true);
+});
+
+
+test("barge-in resets player before new TTS response", () => {
+  const { controller } = makeController();
+
+  let finished = false;
+  let resetCount = 0;
+
+  controller.player = {
+    stop() {
+      finished = true;
+    },
+
+    reset() {
+      finished = false;
+      resetCount++;
+    },
+  };
+
+  controller.turnId = 1;
+  controller.setState("speaking");
+
+  controller.requestBargeIn();
+
+  assert.equal(finished, true);
+
+  controller.handleGatewayEvent({
+    type: "interrupted",
+    turn_id: 1,
+  });
+
+  assert.equal(controller.state, "starting");
+
+  controller.handleGatewayEvent({
+    type: "turn.started",
+    turn_id: 2,
+  });
+
+  assert.equal(controller.state, "recording");
+  assert.equal(finished, false);
+  assert.equal(resetCount, 1);
+});