primeira versão customizada de VAD- Voice Activity Detection
This commit is contained in:
@@ -202,3 +202,24 @@ test("exposes playback analyser", () => {
|
||||
player.analyser
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("late PCM after stop is ignored", async () => {
|
||||
const context = new FakeAudioContext();
|
||||
|
||||
const player = new PCMPlayer({
|
||||
audioContext: context,
|
||||
sampleRate: 48000,
|
||||
});
|
||||
|
||||
player.stop();
|
||||
|
||||
const pcm = new ArrayBuffer(8);
|
||||
|
||||
await assert.doesNotReject(
|
||||
player.enqueue(pcm)
|
||||
);
|
||||
|
||||
assert.equal(player.playing, false);
|
||||
assert.equal(player.sources.size, 0);
|
||||
});
|
||||
|
||||
@@ -29,6 +29,8 @@ class FakeWebSocket {
|
||||
|
||||
globalThis.WebSocket = FakeWebSocket;
|
||||
globalThis.cancelAnimationFrame = () => {};
|
||||
globalThis.requestAnimationFrame = () => 1;
|
||||
|
||||
|
||||
afterEach(() => {
|
||||
// Restore only state modified by individual tests.
|
||||
@@ -145,27 +147,31 @@ test("speech end requests worklet flush", () => {
|
||||
assert.equal(controller.ws.sent.length, 0);
|
||||
});
|
||||
|
||||
|
||||
test("audio.stop is sent after flushed", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("flushing");
|
||||
|
||||
controller.handlePCM(new ArrayBuffer(4));
|
||||
controller.handleFlushed();
|
||||
|
||||
const messages = controller.ws.sent
|
||||
.filter((message) => typeof message === "string")
|
||||
.map((message) => JSON.parse(message));
|
||||
|
||||
assert.ok(
|
||||
messages.some(
|
||||
(message) =>
|
||||
message.type === "audio.stop" &&
|
||||
message.turn_id === 1
|
||||
)
|
||||
);
|
||||
|
||||
assert.equal(controller.state, "processing");
|
||||
|
||||
const lastMessage = JSON.parse(controller.ws.sent.at(-1));
|
||||
|
||||
assert.deepEqual(lastMessage, {
|
||||
type: "audio.stop",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.ws.sent.length, 3);
|
||||
});
|
||||
|
||||
|
||||
test("speech ending before turn.started is handled", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
@@ -337,3 +343,393 @@ test("binary audio is forwarded to PCMPlayer", async () => {
|
||||
|
||||
assert.deepEqual(received, [pcm]);
|
||||
});
|
||||
|
||||
|
||||
|
||||
test("barge-in stops playback and starts a new turn", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let playbackStopped = false;
|
||||
|
||||
controller.player = {
|
||||
stop() {
|
||||
playbackStopped = true;
|
||||
},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
// Simulate user speech detected during TTS.
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(playbackStopped, true);
|
||||
assert.equal(controller.state, "interrupting");
|
||||
assert.equal(controller.bargeInPending, true);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[0]),
|
||||
{
|
||||
type: "interrupt",
|
||||
turn_id: 1,
|
||||
}
|
||||
);
|
||||
|
||||
// Gateway confirms the previous turn was cancelled.
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
assert.equal(controller.ws.closed, false);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[1]),
|
||||
{
|
||||
type: "audio.start",
|
||||
}
|
||||
);
|
||||
|
||||
// Gateway accepts the new turn.
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.turnId, 2);
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.bargeInPending, false);
|
||||
});
|
||||
|
||||
|
||||
|
||||
test("barge-in preserves PCM until new turn starts", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
const pcm1 = new ArrayBuffer(4);
|
||||
const pcm2 = new ArrayBuffer(6);
|
||||
const pcm3 = new ArrayBuffer(8);
|
||||
|
||||
// Audio captured before speech confirmation.
|
||||
controller.handlePCM(pcm1);
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
// Audio captured while interruption is pending.
|
||||
controller.handlePCM(pcm2);
|
||||
|
||||
assert.equal(controller.ws.sent.length, 1);
|
||||
assert.equal(controller.bargeInBuffer.length, 2);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
// Still no PCM sent to the old turn.
|
||||
assert.equal(controller.state, "starting");
|
||||
assert.equal(controller.ws.sent.length, 2);
|
||||
|
||||
// Additional speech arrives before turn.started.
|
||||
controller.handlePCM(pcm3);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
const sent = controller.ws.sent;
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[0]),
|
||||
{ type: "interrupt", turn_id: 1 }
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[1]),
|
||||
{ type: "audio.start" }
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[2]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 0,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(sent[3], pcm1);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[4]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 1,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(sent[5], pcm2);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(sent[6]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 2,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(sent[7], pcm3);
|
||||
|
||||
assert.equal(controller.sequence, 3);
|
||||
assert.equal(controller.state, "recording");
|
||||
});
|
||||
|
||||
|
||||
test("barge-in finishes speech before new turn starts", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
const pcm = new ArrayBuffer(8);
|
||||
|
||||
controller.handlePCM(pcm);
|
||||
controller.requestBargeIn();
|
||||
|
||||
// User finishes speaking before the Gateway responds.
|
||||
controller.finishBargeInSpeech();
|
||||
|
||||
assert.equal(controller.bargeInSpeechEnded, true);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
// Buffered audio must be transmitted before flushing.
|
||||
assert.equal(controller.sequence, 1);
|
||||
|
||||
assert.deepEqual(
|
||||
JSON.parse(controller.ws.sent[2]),
|
||||
{
|
||||
type: "audio.chunk",
|
||||
turn_id: 2,
|
||||
sequence_number: 0,
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(controller.ws.sent[3], pcm);
|
||||
assert.equal(controller.state, "flushing");
|
||||
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
test("barge-in continues recording until speech ends", () => {
|
||||
const { controller, workletMessages } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.bargeInVad.start();
|
||||
controller.requestBargeIn();
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.bargeInRecording, true);
|
||||
assert.equal(controller.bargeInVad.active, true);
|
||||
|
||||
// Speech continues after the new turn starts.
|
||||
const pcm = new ArrayBuffer(8);
|
||||
controller.handlePCM(pcm);
|
||||
|
||||
assert.equal(controller.sequence, 1);
|
||||
|
||||
// Barge-in VAD detects the end of speech.
|
||||
controller.finishBargeInSpeech();
|
||||
|
||||
assert.equal(controller.bargeInRecording, false);
|
||||
assert.equal(controller.state, "flushing");
|
||||
|
||||
assert.deepEqual(
|
||||
workletMessages.at(-1),
|
||||
{ type: "flush" }
|
||||
);
|
||||
|
||||
// Worklet confirms that remaining PCM was emitted.
|
||||
controller.handleFlushed();
|
||||
|
||||
assert.equal(controller.state, "processing");
|
||||
|
||||
const messages = controller.ws.sent
|
||||
.filter((message) => typeof message === "string")
|
||||
.map((message) => JSON.parse(message));
|
||||
|
||||
assert.ok(
|
||||
messages.some(
|
||||
(message) =>
|
||||
message.type === "audio.stop" &&
|
||||
message.turn_id === 2
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
test("late INVALID_TURN recovers barge-in session", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
controller.player = {
|
||||
stop() {},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(controller.state, "interrupting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "error",
|
||||
code: "INVALID_TURN",
|
||||
message: "Turn 1 is no longer active.",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
assert.equal(controller.ws.closed, false);
|
||||
|
||||
const messages = controller.ws.sent
|
||||
.filter((message) => typeof message === "string")
|
||||
.map((message) => JSON.parse(message));
|
||||
|
||||
assert.deepEqual(
|
||||
messages.map((message) => message.type),
|
||||
["interrupt", "audio.start"]
|
||||
);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(controller.turnId, 2);
|
||||
});
|
||||
|
||||
|
||||
test("late TTS chunk during barge-in does not close session", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let stopped = false;
|
||||
|
||||
controller.player = {
|
||||
stop() {
|
||||
stopped = true;
|
||||
},
|
||||
reset() {},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(stopped, true);
|
||||
assert.equal(controller.state, "interrupting");
|
||||
|
||||
// A late PCM chunk from the interrupted TTS.
|
||||
controller.handleAudioChunk(new ArrayBuffer(8));
|
||||
|
||||
// The WebSocket session must remain alive.
|
||||
assert.equal(controller.ws.closed, false);
|
||||
assert.equal(controller.state, "interrupting");
|
||||
assert.equal(controller.bargeInPending, true);
|
||||
});
|
||||
|
||||
|
||||
test("barge-in resets player before new TTS response", () => {
|
||||
const { controller } = makeController();
|
||||
|
||||
let finished = false;
|
||||
let resetCount = 0;
|
||||
|
||||
controller.player = {
|
||||
stop() {
|
||||
finished = true;
|
||||
},
|
||||
|
||||
reset() {
|
||||
finished = false;
|
||||
resetCount++;
|
||||
},
|
||||
};
|
||||
|
||||
controller.turnId = 1;
|
||||
controller.setState("speaking");
|
||||
|
||||
controller.requestBargeIn();
|
||||
|
||||
assert.equal(finished, true);
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "interrupted",
|
||||
turn_id: 1,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "starting");
|
||||
|
||||
controller.handleGatewayEvent({
|
||||
type: "turn.started",
|
||||
turn_id: 2,
|
||||
});
|
||||
|
||||
assert.equal(controller.state, "recording");
|
||||
assert.equal(finished, false);
|
||||
assert.equal(resetCount, 1);
|
||||
});
|
||||
|
||||
343
tests/test_deep_agent.py
Normal file
343
tests/test_deep_agent.py
Normal file
@@ -0,0 +1,343 @@
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from voice_assistant.agent.adapter import AgentAdapter
|
||||
|
||||
|
||||
def test_deep_agent_implements_port():
|
||||
from voice_assistant.agent.deep_agent import DeepAgentsAdapter
|
||||
|
||||
assert issubclass(DeepAgentsAdapter, AgentAdapter)
|
||||
assert callable(getattr(DeepAgentsAdapter, "stream", None))
|
||||
assert callable(getattr(DeepAgentsAdapter, "aclose", None))
|
||||
|
||||
|
||||
def test_deep_agent_session_config():
|
||||
from voice_assistant.agent.deep_agent import DeepAgentsAdapter
|
||||
|
||||
adapter = object.__new__(DeepAgentsAdapter)
|
||||
|
||||
assert adapter._config("session-a") == {
|
||||
"configurable": {
|
||||
"thread_id": "session-a",
|
||||
}
|
||||
}
|
||||
|
||||
assert adapter._config("session-b") == {
|
||||
"configurable": {
|
||||
"thread_id": "session-b",
|
||||
}
|
||||
}
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
adapter._config("")
|
||||
|
||||
|
||||
|
||||
def test_deep_agent_cleanup_is_idempotent():
|
||||
import asyncio
|
||||
|
||||
from voice_assistant.agent.deep_agent import DeepAgentsAdapter
|
||||
|
||||
adapter = object.__new__(DeepAgentsAdapter)
|
||||
adapter.model = MagicMock()
|
||||
adapter.model.aclose = AsyncMock()
|
||||
adapter._closed = False
|
||||
|
||||
async def scenario():
|
||||
await adapter.aclose()
|
||||
await adapter.aclose()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
adapter.model.aclose.assert_awaited_once()
|
||||
assert adapter._closed is True
|
||||
|
||||
|
||||
|
||||
def test_deep_agent_stream_can_be_cancelled():
|
||||
import asyncio
|
||||
|
||||
from voice_assistant.agent.deep_agent import DeepAgentsAdapter
|
||||
|
||||
async def scenario():
|
||||
adapter = object.__new__(DeepAgentsAdapter)
|
||||
adapter._closed = False
|
||||
|
||||
started = asyncio.Event()
|
||||
cancelled = asyncio.Event()
|
||||
|
||||
class FakeAgent:
|
||||
async def astream(self, *args, **kwargs):
|
||||
started.set()
|
||||
|
||||
try:
|
||||
yield (
|
||||
FakeMessage("Primeiro fragmento"),
|
||||
{"langgraph_node": "model"},
|
||||
)
|
||||
|
||||
await asyncio.Event().wait()
|
||||
|
||||
except asyncio.CancelledError:
|
||||
cancelled.set()
|
||||
raise
|
||||
|
||||
class FakeMessage:
|
||||
def __init__(self, content):
|
||||
self.content = content
|
||||
self.tool_call_chunks = []
|
||||
|
||||
adapter.agent = FakeAgent()
|
||||
|
||||
async def consume():
|
||||
async for _ in adapter.stream(
|
||||
session_id="session-cancel",
|
||||
turn_id=1,
|
||||
text="Teste de cancelamento",
|
||||
):
|
||||
pass
|
||||
|
||||
task = asyncio.create_task(consume())
|
||||
|
||||
await asyncio.wait_for(started.wait(), timeout=2.0)
|
||||
|
||||
task.cancel()
|
||||
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
assert cancelled.is_set()
|
||||
assert task.done()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_deep_agent_isolates_session_history():
|
||||
import asyncio
|
||||
|
||||
from deepagents import create_deep_agent
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langchain_core.language_models import FakeListChatModel
|
||||
from langgraph.checkpoint.memory import MemorySaver
|
||||
|
||||
async def scenario():
|
||||
checkpointer = MemorySaver()
|
||||
|
||||
class ToolCompatibleFakeChatModel(FakeListChatModel):
|
||||
def bind_tools(self, tools, *, tool_choice=None, **kwargs):
|
||||
return self
|
||||
|
||||
|
||||
model = ToolCompatibleFakeChatModel(
|
||||
responses=[
|
||||
"Resposta A1",
|
||||
"Resposta B1",
|
||||
"Resposta A2",
|
||||
]
|
||||
)
|
||||
|
||||
agent = create_deep_agent(
|
||||
model=model,
|
||||
tools=[],
|
||||
system_prompt="Responda de forma breve.",
|
||||
checkpointer=checkpointer,
|
||||
)
|
||||
|
||||
async def ask(session_id, text):
|
||||
return await agent.ainvoke(
|
||||
{
|
||||
"messages": [
|
||||
HumanMessage(content=text)
|
||||
]
|
||||
},
|
||||
config={
|
||||
"configurable": {
|
||||
"thread_id": session_id,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
await ask("session-a", "Destino: Japão")
|
||||
await ask("session-b", "Destino: Canadá")
|
||||
await ask("session-a", "Qual era meu destino?")
|
||||
|
||||
def history(session_id):
|
||||
snapshot = agent.get_state({
|
||||
"configurable": {
|
||||
"thread_id": session_id,
|
||||
}
|
||||
})
|
||||
|
||||
return snapshot.values.get("messages", [])
|
||||
|
||||
history_a = history("session-a")
|
||||
history_b = history("session-b")
|
||||
|
||||
texts_a = [
|
||||
message.content
|
||||
for message in history_a
|
||||
if isinstance(message, HumanMessage)
|
||||
]
|
||||
|
||||
texts_b = [
|
||||
message.content
|
||||
for message in history_b
|
||||
if isinstance(message, HumanMessage)
|
||||
]
|
||||
|
||||
assert texts_a == [
|
||||
"Destino: Japão",
|
||||
"Qual era meu destino?",
|
||||
]
|
||||
|
||||
assert texts_b == [
|
||||
"Destino: Canadá",
|
||||
]
|
||||
|
||||
assert all(
|
||||
"Canadá" not in text
|
||||
for text in texts_a
|
||||
)
|
||||
|
||||
assert all(
|
||||
"Japão" not in text
|
||||
for text in texts_b
|
||||
)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_deep_agent_checkpoint_survives_cancellation():
|
||||
import asyncio
|
||||
|
||||
from deepagents import create_deep_agent
|
||||
from langchain_core.language_models import FakeListChatModel
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langgraph.checkpoint.memory import MemorySaver
|
||||
|
||||
class CancellableFakeModel(FakeListChatModel):
|
||||
def bind_tools(self, tools, *, tool_choice=None, **kwargs):
|
||||
return self
|
||||
|
||||
async def _agenerate(
|
||||
self,
|
||||
messages,
|
||||
stop=None,
|
||||
run_manager=None,
|
||||
**kwargs,
|
||||
):
|
||||
last_user_message = next(
|
||||
(
|
||||
message
|
||||
for message in reversed(messages)
|
||||
if isinstance(message, HumanMessage)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
if (
|
||||
last_user_message is not None
|
||||
and last_user_message.content == "Interromper esta resposta"
|
||||
):
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
return await super()._agenerate(
|
||||
messages,
|
||||
stop=stop,
|
||||
run_manager=run_manager,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
async def scenario():
|
||||
nonlocal started
|
||||
|
||||
started = asyncio.Event()
|
||||
|
||||
model = CancellableFakeModel(
|
||||
responses=[
|
||||
"Contexto inicial confirmado.",
|
||||
"Contexto preservado.",
|
||||
]
|
||||
)
|
||||
|
||||
agent = create_deep_agent(
|
||||
model=model,
|
||||
tools=[],
|
||||
system_prompt="Responda brevemente.",
|
||||
checkpointer=MemorySaver(),
|
||||
)
|
||||
|
||||
config = {
|
||||
"configurable": {
|
||||
"thread_id": "session-cancellation-test",
|
||||
}
|
||||
}
|
||||
|
||||
async def ask(text):
|
||||
return await agent.ainvoke(
|
||||
{
|
||||
"messages": [
|
||||
HumanMessage(content=text)
|
||||
]
|
||||
},
|
||||
config=config,
|
||||
)
|
||||
|
||||
# First turn completes successfully.
|
||||
await ask("Meu destino é Japão")
|
||||
|
||||
# Second turn begins but never completes.
|
||||
task = asyncio.create_task(
|
||||
ask("Interromper esta resposta")
|
||||
)
|
||||
|
||||
await asyncio.wait_for(
|
||||
started.wait(),
|
||||
timeout=2.0,
|
||||
)
|
||||
|
||||
task.cancel()
|
||||
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
assert task.done()
|
||||
assert task.cancelled()
|
||||
|
||||
# The same thread must remain usable.
|
||||
await asyncio.wait_for(
|
||||
ask("Qual era meu destino?"),
|
||||
timeout=5.0,
|
||||
)
|
||||
|
||||
snapshot = await agent.aget_state(config)
|
||||
|
||||
messages = snapshot.values.get("messages", [])
|
||||
|
||||
human_messages = [
|
||||
message.content
|
||||
for message in messages
|
||||
if isinstance(message, HumanMessage)
|
||||
]
|
||||
|
||||
assert "Meu destino é Japão" in human_messages
|
||||
assert "Qual era meu destino?" in human_messages
|
||||
|
||||
# An interrupted execution must not be recorded
|
||||
# as a completed assistant response.
|
||||
assert not any(
|
||||
getattr(message, "content", None)
|
||||
== "Resposta interrompida concluída."
|
||||
for message in messages
|
||||
)
|
||||
|
||||
started = None
|
||||
asyncio.run(scenario())
|
||||
|
||||
@@ -348,6 +348,142 @@ def test_websocket_session_end():
|
||||
assert session_ended["session_id"] == session_id
|
||||
|
||||
|
||||
def test_websocket_session_end_closes_agent_provider():
|
||||
"""
|
||||
Explicit session.end must close the Agent provider.
|
||||
"""
|
||||
agent = MockAgentAdapter()
|
||||
agent.aclose = AsyncMock()
|
||||
|
||||
with patch.object(
|
||||
main,
|
||||
"create_agent_adapter",
|
||||
return_value=agent,
|
||||
):
|
||||
with client.websocket_connect("/ws/audio") as websocket:
|
||||
websocket.send_json({
|
||||
"type": "session.start",
|
||||
"audio": {
|
||||
"encoding": AUDIO_ENCODING,
|
||||
"sample_rate": AUDIO_SAMPLE_RATE,
|
||||
"channels": AUDIO_CHANNELS,
|
||||
},
|
||||
})
|
||||
|
||||
ready = websocket.receive_json()
|
||||
assert ready["type"] == "session.ready"
|
||||
|
||||
websocket.send_json({
|
||||
"type": "session.end",
|
||||
})
|
||||
|
||||
ended = websocket.receive_json()
|
||||
assert ended["type"] == "session.ended"
|
||||
|
||||
agent.aclose.assert_awaited_once()
|
||||
|
||||
|
||||
|
||||
def test_websocket_disconnect_closes_agent_provider():
|
||||
"""
|
||||
Unexpected WebSocket disconnect must close the Agent provider.
|
||||
"""
|
||||
agent = MockAgentAdapter()
|
||||
agent.aclose = AsyncMock()
|
||||
|
||||
with patch.object(
|
||||
main,
|
||||
"create_agent_adapter",
|
||||
return_value=agent,
|
||||
):
|
||||
with client.websocket_connect("/ws/audio") as websocket:
|
||||
websocket.send_json({
|
||||
"type": "session.start",
|
||||
"audio": {
|
||||
"encoding": AUDIO_ENCODING,
|
||||
"sample_rate": AUDIO_SAMPLE_RATE,
|
||||
"channels": AUDIO_CHANNELS,
|
||||
},
|
||||
})
|
||||
|
||||
ready = websocket.receive_json()
|
||||
assert ready["type"] == "session.ready"
|
||||
|
||||
# Leave without sending session.end.
|
||||
# TestClient closes the WebSocket context.
|
||||
|
||||
agent.aclose.assert_awaited_once()
|
||||
|
||||
|
||||
def test_websocket_interrupt_preserves_agent_provider():
|
||||
"""
|
||||
Interrupt must cancel the turn without closing the Agent provider.
|
||||
The provider is closed only when the WebSocket session ends.
|
||||
"""
|
||||
created_stt.clear()
|
||||
|
||||
agent = MockAgentAdapter()
|
||||
agent.aclose = AsyncMock()
|
||||
|
||||
with patch.object(
|
||||
main,
|
||||
"create_agent_adapter",
|
||||
return_value=agent,
|
||||
):
|
||||
with client.websocket_connect("/ws/audio") as websocket:
|
||||
websocket.send_json({
|
||||
"type": "session.start",
|
||||
"audio": {
|
||||
"encoding": AUDIO_ENCODING,
|
||||
"sample_rate": AUDIO_SAMPLE_RATE,
|
||||
"channels": AUDIO_CHANNELS,
|
||||
},
|
||||
})
|
||||
|
||||
ready = websocket.receive_json()
|
||||
assert ready["type"] == "session.ready"
|
||||
|
||||
websocket.send_json({
|
||||
"type": "audio.start",
|
||||
})
|
||||
|
||||
started = websocket.receive_json()
|
||||
assert started["type"] == "turn.started"
|
||||
|
||||
websocket.send_json({
|
||||
"type": "interrupt",
|
||||
"turn_id": started["turn_id"],
|
||||
})
|
||||
|
||||
interrupted = websocket.receive_json()
|
||||
assert interrupted["type"] == "interrupted"
|
||||
|
||||
# Interrupt must not close the provider.
|
||||
agent.aclose.assert_not_awaited()
|
||||
|
||||
# The same WebSocket must accept a new turn.
|
||||
websocket.send_json({
|
||||
"type": "audio.start",
|
||||
})
|
||||
|
||||
next_turn = websocket.receive_json()
|
||||
assert next_turn["type"] == "turn.started"
|
||||
assert next_turn["turn_id"] != started["turn_id"]
|
||||
|
||||
# Provider must remain open.
|
||||
agent.aclose.assert_not_awaited()
|
||||
|
||||
websocket.send_json({
|
||||
"type": "session.end",
|
||||
})
|
||||
|
||||
ended = websocket.receive_json()
|
||||
assert ended["type"] == "session.ended"
|
||||
|
||||
agent.aclose.assert_awaited_once()
|
||||
|
||||
|
||||
|
||||
def test_websocket_forwards_stt_events():
|
||||
created_stt.clear()
|
||||
created_turn_managers.clear()
|
||||
@@ -1800,3 +1936,477 @@ def test_websocket_streaming_agent_and_tts():
|
||||
|
||||
turn_ended = websocket.receive_json()
|
||||
assert turn_ended["type"] == "turn.ended"
|
||||
|
||||
|
||||
def test_websocket_interrupt_during_streaming_tts():
|
||||
"""
|
||||
Verify that interrupt cancels an active streaming TTS turn
|
||||
while normal turn finalization is pending.
|
||||
"""
|
||||
created_stt.clear()
|
||||
|
||||
tts_started = threading.Event()
|
||||
tts_cancelled = threading.Event()
|
||||
|
||||
class SlowStreamingAgent:
|
||||
async def stream(self, session_id, turn_id, text):
|
||||
yield "Olá, esta é uma resposta longa."
|
||||
|
||||
class SlowStreamingTTS:
|
||||
async def synthesize(self, text):
|
||||
try:
|
||||
tts_started.set()
|
||||
|
||||
yield b"\x01\x02\x03\x04"
|
||||
|
||||
# Keep synthesis active until cancellation.
|
||||
await asyncio.Event().wait()
|
||||
|
||||
except asyncio.CancelledError:
|
||||
tts_cancelled.set()
|
||||
raise
|
||||
|
||||
with patch.object(
|
||||
main,
|
||||
"create_agent_adapter",
|
||||
return_value=SlowStreamingAgent(),
|
||||
):
|
||||
with patch.object(
|
||||
main,
|
||||
"create_tts_adapter",
|
||||
return_value=SlowStreamingTTS(),
|
||||
):
|
||||
with client.websocket_connect("/ws/audio") as websocket:
|
||||
websocket.send_json({
|
||||
"type": "session.start",
|
||||
"audio": {
|
||||
"encoding": AUDIO_ENCODING,
|
||||
"sample_rate": AUDIO_SAMPLE_RATE,
|
||||
"channels": AUDIO_CHANNELS,
|
||||
},
|
||||
})
|
||||
|
||||
ready = websocket.receive_json()
|
||||
assert ready["type"] == "session.ready"
|
||||
|
||||
websocket.send_json({"type": "audio.start"})
|
||||
|
||||
started = websocket.receive_json()
|
||||
assert started["type"] == "turn.started"
|
||||
|
||||
turn_id = started["turn_id"]
|
||||
|
||||
stt = created_stt[-1]
|
||||
stt.emit_final_now("Olá")
|
||||
|
||||
# Wait until the Gateway delivers the first TTS audio chunk.
|
||||
first_audio_received = False
|
||||
|
||||
for _ in range(20):
|
||||
message = websocket.receive()
|
||||
|
||||
if message.get("bytes") is not None:
|
||||
assert message["bytes"] == b"\x01\x02\x03\x04"
|
||||
first_audio_received = True
|
||||
break
|
||||
|
||||
import json
|
||||
event = json.loads(message["text"])
|
||||
|
||||
if event["type"] == "error":
|
||||
raise AssertionError(
|
||||
f"Gateway returned error: {event}"
|
||||
)
|
||||
|
||||
assert first_audio_received
|
||||
assert tts_started.is_set()
|
||||
|
||||
websocket.send_json({
|
||||
"type": "audio.stop",
|
||||
"turn_id": turn_id,
|
||||
})
|
||||
|
||||
websocket.send_json({
|
||||
"type": "interrupt",
|
||||
"turn_id": turn_id,
|
||||
})
|
||||
|
||||
# Streaming events may precede interruption.
|
||||
for _ in range(20):
|
||||
message = websocket.receive()
|
||||
|
||||
if message.get("bytes") is not None:
|
||||
continue
|
||||
|
||||
import json
|
||||
event = json.loads(message["text"])
|
||||
|
||||
if event["type"] == "interrupted":
|
||||
assert event["turn_id"] == turn_id
|
||||
break
|
||||
|
||||
if event["type"] == "error":
|
||||
raise AssertionError(
|
||||
f"Gateway returned error: {event}"
|
||||
)
|
||||
else:
|
||||
raise AssertionError(
|
||||
"Gateway did not confirm interruption."
|
||||
)
|
||||
|
||||
assert tts_cancelled.wait(timeout=2.0)
|
||||
|
||||
# A new turn must be accepted in the same session.
|
||||
websocket.send_json({"type": "audio.start"})
|
||||
|
||||
next_turn = websocket.receive_json()
|
||||
|
||||
assert next_turn["type"] == "turn.started"
|
||||
assert next_turn["turn_id"] != turn_id
|
||||
|
||||
websocket.send_json({
|
||||
"type": "interrupt",
|
||||
"turn_id": next_turn["turn_id"],
|
||||
})
|
||||
|
||||
interrupted = websocket.receive_json()
|
||||
assert interrupted["type"] == "interrupted"
|
||||
|
||||
|
||||
|
||||
def test_websocket_session_end_during_streaming_tts():
|
||||
"""
|
||||
Verify session.end cancels an active streaming TTS turn.
|
||||
|
||||
The Gateway must cancel provider execution before
|
||||
acknowledging session closure.
|
||||
"""
|
||||
import json
|
||||
|
||||
created_stt.clear()
|
||||
|
||||
tts_cancelled = threading.Event()
|
||||
|
||||
class SlowStreamingAgent:
|
||||
async def stream(self, session_id, turn_id, text):
|
||||
yield "Esta é uma resposta longa."
|
||||
|
||||
class SlowStreamingTTS:
|
||||
async def synthesize(self, text):
|
||||
try:
|
||||
yield b"\x01\x02\x03\x04"
|
||||
|
||||
# Keep synthesis running until cancelled.
|
||||
await asyncio.Event().wait()
|
||||
|
||||
except asyncio.CancelledError:
|
||||
tts_cancelled.set()
|
||||
raise
|
||||
|
||||
with patch.object(
|
||||
main,
|
||||
"create_agent_adapter",
|
||||
return_value=SlowStreamingAgent(),
|
||||
):
|
||||
with patch.object(
|
||||
main,
|
||||
"create_tts_adapter",
|
||||
return_value=SlowStreamingTTS(),
|
||||
):
|
||||
with client.websocket_connect("/ws/audio") as websocket:
|
||||
websocket.send_json({
|
||||
"type": "session.start",
|
||||
"audio": {
|
||||
"encoding": AUDIO_ENCODING,
|
||||
"sample_rate": AUDIO_SAMPLE_RATE,
|
||||
"channels": AUDIO_CHANNELS,
|
||||
},
|
||||
})
|
||||
|
||||
ready = websocket.receive_json()
|
||||
assert ready["type"] == "session.ready"
|
||||
|
||||
websocket.send_json({"type": "audio.start"})
|
||||
|
||||
started = websocket.receive_json()
|
||||
assert started["type"] == "turn.started"
|
||||
|
||||
turn_id = started["turn_id"]
|
||||
|
||||
stt = created_stt[-1]
|
||||
stt.emit_final_now("Olá")
|
||||
|
||||
# Wait until TTS audio reaches the client.
|
||||
first_audio_received = False
|
||||
|
||||
for _ in range(20):
|
||||
message = websocket.receive()
|
||||
|
||||
if message.get("bytes") is not None:
|
||||
assert message["bytes"] == b"\x01\x02\x03\x04"
|
||||
first_audio_received = True
|
||||
break
|
||||
|
||||
event = json.loads(message["text"])
|
||||
|
||||
if event["type"] == "error":
|
||||
raise AssertionError(
|
||||
f"Gateway returned error: {event}"
|
||||
)
|
||||
|
||||
assert first_audio_received
|
||||
|
||||
websocket.send_json({
|
||||
"type": "audio.stop",
|
||||
"turn_id": turn_id,
|
||||
})
|
||||
|
||||
websocket.send_json({
|
||||
"type": "session.end",
|
||||
})
|
||||
|
||||
# Streaming events may already be queued.
|
||||
for _ in range(20):
|
||||
message = websocket.receive()
|
||||
|
||||
if message.get("bytes") is not None:
|
||||
continue
|
||||
|
||||
event = json.loads(message["text"])
|
||||
|
||||
if event["type"] == "session.ended":
|
||||
assert event["session_id"] == ready["session_id"]
|
||||
break
|
||||
|
||||
if event["type"] == "turn.ended":
|
||||
raise AssertionError(
|
||||
"Turn completed normally during session shutdown."
|
||||
)
|
||||
|
||||
if event["type"] == "error":
|
||||
raise AssertionError(
|
||||
f"Gateway returned error: {event}"
|
||||
)
|
||||
else:
|
||||
raise AssertionError(
|
||||
"Gateway did not confirm session closure."
|
||||
)
|
||||
|
||||
assert tts_cancelled.wait(timeout=2.0)
|
||||
|
||||
|
||||
|
||||
|
||||
def test_websocket_interrupt_after_turn_completed():
|
||||
"""
|
||||
Verify behavior when interrupt arrives after turn.ended.
|
||||
|
||||
A completed turn must not be cancelled again.
|
||||
"""
|
||||
created_stt.clear()
|
||||
|
||||
with client.websocket_connect("/ws/audio") as websocket:
|
||||
websocket.send_json({
|
||||
"type": "session.start",
|
||||
"audio": {
|
||||
"encoding": AUDIO_ENCODING,
|
||||
"sample_rate": AUDIO_SAMPLE_RATE,
|
||||
"channels": AUDIO_CHANNELS,
|
||||
},
|
||||
})
|
||||
|
||||
ready = websocket.receive_json()
|
||||
assert ready["type"] == "session.ready"
|
||||
|
||||
websocket.send_json({"type": "audio.start"})
|
||||
|
||||
started = websocket.receive_json()
|
||||
assert started["type"] == "turn.started"
|
||||
|
||||
turn_id = started["turn_id"]
|
||||
|
||||
stt = created_stt[-1]
|
||||
stt.emit_final_now("Olá")
|
||||
|
||||
# Consume events until the first turn finishes.
|
||||
for _ in range(30):
|
||||
message = websocket.receive()
|
||||
|
||||
if message.get("bytes") is not None:
|
||||
continue
|
||||
|
||||
import json
|
||||
event = json.loads(message["text"])
|
||||
|
||||
if event["type"] == "error":
|
||||
raise AssertionError(
|
||||
f"Gateway returned error: {event}"
|
||||
)
|
||||
|
||||
if event["type"] == "stt.final":
|
||||
websocket.send_json({
|
||||
"type": "audio.stop",
|
||||
"turn_id": turn_id,
|
||||
})
|
||||
|
||||
if event["type"] == "turn.ended":
|
||||
break
|
||||
else:
|
||||
raise AssertionError(
|
||||
"Turn did not complete normally."
|
||||
)
|
||||
|
||||
# Interrupt the already completed turn.
|
||||
websocket.send_json({
|
||||
"type": "interrupt",
|
||||
"turn_id": turn_id,
|
||||
})
|
||||
|
||||
response = websocket.receive_json()
|
||||
|
||||
# Existing Gateway behavior may acknowledge stale IDs.
|
||||
# Capture the current contract before changing it.
|
||||
assert response["type"] == "error"
|
||||
assert response["code"] == "INVALID_TURN"
|
||||
assert response["turn_id"] == turn_id
|
||||
|
||||
|
||||
def test_websocket_interrupt_wins_pending_tts_completion():
|
||||
"""
|
||||
Verify interruption wins while TTS completion is pending.
|
||||
|
||||
The interrupted turn must not emit turn.ended,
|
||||
and a new turn must be accepted.
|
||||
"""
|
||||
import json
|
||||
|
||||
created_stt.clear()
|
||||
|
||||
tts_cancelled = threading.Event()
|
||||
|
||||
class ControlledAgent:
|
||||
async def stream(self, session_id, turn_id, text):
|
||||
yield "Esta resposta será interrompida."
|
||||
|
||||
class ControlledTTS:
|
||||
async def synthesize(self, text):
|
||||
try:
|
||||
yield b"\x01\x02\x03\x04"
|
||||
|
||||
# Completion remains pending until cancellation.
|
||||
await asyncio.Event().wait()
|
||||
|
||||
except asyncio.CancelledError:
|
||||
tts_cancelled.set()
|
||||
raise
|
||||
|
||||
with patch.object(
|
||||
main,
|
||||
"create_agent_adapter",
|
||||
return_value=ControlledAgent(),
|
||||
):
|
||||
with patch.object(
|
||||
main,
|
||||
"create_tts_adapter",
|
||||
return_value=ControlledTTS(),
|
||||
):
|
||||
with client.websocket_connect("/ws/audio") as websocket:
|
||||
websocket.send_json({
|
||||
"type": "session.start",
|
||||
"audio": {
|
||||
"encoding": AUDIO_ENCODING,
|
||||
"sample_rate": AUDIO_SAMPLE_RATE,
|
||||
"channels": AUDIO_CHANNELS,
|
||||
},
|
||||
})
|
||||
|
||||
ready = websocket.receive_json()
|
||||
assert ready["type"] == "session.ready"
|
||||
|
||||
websocket.send_json({
|
||||
"type": "audio.start",
|
||||
})
|
||||
|
||||
started = websocket.receive_json()
|
||||
assert started["type"] == "turn.started"
|
||||
|
||||
turn_id = started["turn_id"]
|
||||
|
||||
stt = created_stt[-1]
|
||||
stt.emit_final_now("Olá")
|
||||
|
||||
# Observe the first PCM chunk.
|
||||
for _ in range(20):
|
||||
message = websocket.receive()
|
||||
|
||||
if message.get("bytes") is not None:
|
||||
assert message["bytes"] == b"\x01\x02\x03\x04"
|
||||
break
|
||||
|
||||
event = json.loads(message["text"])
|
||||
|
||||
if event["type"] == "error":
|
||||
raise AssertionError(event)
|
||||
else:
|
||||
raise AssertionError(
|
||||
"TTS did not produce its first chunk."
|
||||
)
|
||||
|
||||
websocket.send_json({
|
||||
"type": "audio.stop",
|
||||
"turn_id": turn_id,
|
||||
})
|
||||
|
||||
websocket.send_json({
|
||||
"type": "interrupt",
|
||||
"turn_id": turn_id,
|
||||
})
|
||||
|
||||
terminal_events = []
|
||||
|
||||
for _ in range(20):
|
||||
message = websocket.receive()
|
||||
|
||||
if message.get("bytes") is not None:
|
||||
continue
|
||||
|
||||
event = json.loads(message["text"])
|
||||
|
||||
if event["type"] in {
|
||||
"turn.ended",
|
||||
"interrupted",
|
||||
}:
|
||||
terminal_events.append(event["type"])
|
||||
|
||||
if event["type"] == "error":
|
||||
raise AssertionError(event)
|
||||
|
||||
if event["type"] == "interrupted":
|
||||
break
|
||||
else:
|
||||
raise AssertionError(
|
||||
"Interruption was not acknowledged."
|
||||
)
|
||||
|
||||
assert terminal_events == ["interrupted"]
|
||||
assert tts_cancelled.wait(timeout=2.0)
|
||||
|
||||
# Confirm that the session accepts another turn.
|
||||
websocket.send_json({
|
||||
"type": "audio.start",
|
||||
})
|
||||
|
||||
next_turn = websocket.receive_json()
|
||||
|
||||
assert next_turn["type"] == "turn.started"
|
||||
assert next_turn["turn_id"] != turn_id
|
||||
|
||||
websocket.send_json({
|
||||
"type": "interrupt",
|
||||
"turn_id": next_turn["turn_id"],
|
||||
})
|
||||
|
||||
response = websocket.receive_json()
|
||||
|
||||
assert response["type"] == "interrupted"
|
||||
assert response["turn_id"] == next_turn["turn_id"]
|
||||
|
||||
Reference in New Issue
Block a user