primeira versão customizada de VAD- Voice Activity Detection

This commit is contained in:
2026-10-30 19:15:01 +00:00
parent 2449b34137
commit aba05cb826
14 changed files with 3457 additions and 129 deletions

View File

@@ -33,3 +33,10 @@ providers:
capabilities: capabilities:
realtime: true realtime: true
streaming: true streaming: true
deepagents:
name: "LangChain Deep Agents"
implementation: "voice_assistant.agent.deep_agent:DeepAgentsAdapter"
capabilities:
realtime: true
streaming: true

View File

@@ -5,10 +5,12 @@ description = "Realtime voice assistant with WebSocket audio gateway, streaming
readme = "README.md" readme = "README.md"
requires-python = ">=3.12" requires-python = ">=3.12"
dependencies = [ dependencies = [
"deepagents>=0.7.26",
"fastapi>=0.141.1", "fastapi>=0.141.1",
"httpx>=0.28,<1", "httpx>=0.28,<1",
"httpx2>=2.13.1", "httpx2>=2.13.1",
"ipykernel>=7.3.0", "ipykernel>=7.3.0",
"langchain-oci>=0.3.2",
"nvidia-riva-client==2.26.0", "nvidia-riva-client==2.26.0",
"oci>=2.187.2", "oci>=2.187.2",
"pytest>=9.1.1", "pytest>=9.1.1",

View File

@@ -0,0 +1,173 @@
"""
Deep Agents implementation of the AgentVoice Agent port.
Responsibilities:
- OCI Generative AI inference through LangChain.
- Contextual memory per session.
- Text streaming compatible with VoicePipeline.
- Asynchronous resource cleanup.
Does not manage:
- WebSocket connections.
- STT or TTS.
- Semantic turn lifecycle.
"""
from __future__ import annotations
from collections.abc import AsyncIterator
from contextlib import aclosing
from deepagents import create_deep_agent
from langchain_core.messages import HumanMessage
from langchain_oci import ChatOCIGenAI
from langgraph.checkpoint.memory import MemorySaver
from voice_assistant.agent.adapter import AgentAdapter
DEFAULT_SYSTEM_PROMPT = """
Você é o AgentVoice, um assistente pessoal por voz. Que responde pelo nome de Roberta.
Responda em português brasileiro, de forma natural,
objetiva e adequada para reprodução por voz.
Mantenha o contexto das mensagens anteriores.
Interprete correções e complementos do usuário
considerando o histórico da conversa.
Não invente informações atuais ou resultados
de ferramentas que não foram executadas.
"""
class DeepAgentsAdapter(AgentAdapter):
def __init__(
self,
region: str,
compartment_id: str,
model_id: str,
temperature: float,
max_completion_tokens: int,
timeout_seconds: float,
system_prompt: str = DEFAULT_SYSTEM_PROMPT,
) -> None:
if not region:
raise ValueError("OCI region is required.")
if not compartment_id:
raise ValueError("OCI compartment_id is required.")
if not model_id:
raise ValueError("OCI model_id is required.")
if timeout_seconds <= 0:
raise ValueError("timeout_seconds must be positive.")
self.model = ChatOCIGenAI(
model_id=model_id,
compartment_id=compartment_id,
service_endpoint=(
f"https://inference.generativeai."
f"{region}.oci.oraclecloud.com"
),
auth_type="INSTANCE_PRINCIPAL",
is_stream=True,
model_kwargs={
"temperature": temperature,
"max_tokens": max_completion_tokens,
},
)
self.checkpointer = MemorySaver()
self.agent = create_deep_agent(
model=self.model,
tools=[],
system_prompt=system_prompt,
checkpointer=self.checkpointer,
)
self._closed = False
def _config(self, session_id: str) -> dict:
if not session_id:
raise ValueError("session_id is required.")
return {
"configurable": {
"thread_id": session_id,
}
}
async def process(
self,
session_id: str,
turn_id: int,
text: str,
) -> str:
parts: list[str] = []
async with aclosing(
self.stream(
session_id=session_id,
turn_id=turn_id,
text=text,
)
) as chunks:
async for chunk in chunks:
parts.append(chunk)
response = "".join(parts).strip()
if not response:
raise RuntimeError(
"Deep Agent returned an empty response."
)
return response
async def stream(
self,
session_id: str,
turn_id: int,
text: str,
) -> AsyncIterator[str]:
if self._closed:
raise RuntimeError("Deep Agent is closed.")
if not text.strip():
raise ValueError("Agent input text cannot be empty.")
config = self._config(session_id)
async for message, metadata in self.agent.astream(
{
"messages": [
HumanMessage(content=text)
]
},
config=config,
stream_mode="messages",
):
# Ignore tools and subagent outputs.
if metadata.get("langgraph_node") != "model":
continue
# Ignore tool-call fragments.
if getattr(message, "tool_call_chunks", None):
continue
content = message.content
if isinstance(content, str) and content:
yield content
async def aclose(self) -> None:
if self._closed:
return
self._closed = True
await self.model.aclose()

View File

@@ -155,6 +155,20 @@ async def send_json(
) )
) )
async def close_optional_provider(provider: object | None) -> None:
"""
Close providers exposing an optional asynchronous cleanup hook.
Providers without aclose() require no action.
"""
if provider is None:
return
close_method = getattr(provider, "aclose", None)
if callable(close_method):
await close_method()
async def forward_stt_events( async def forward_stt_events(
websocket: WebSocket, websocket: WebSocket,
@@ -331,13 +345,29 @@ async def audio_socket(websocket: WebSocket) -> None:
except asyncio.CancelledError: except asyncio.CancelledError:
raise raise
except Exception: except Exception as exc:
logger.exception( logger.exception(
"Turn finalization failed. session_id=%s turn_id=%s", "Turn finalization failed. session_id=%s turn_id=%s",
session.session_id, session.session_id,
turn_id, turn_id,
) )
if turn_id not in interrupting_turns:
try:
await send_json(
websocket,
error_event(
"TURN_EXECUTION_FAILED",
str(exc),
session_id=session.session_id,
turn_id=turn_id,
),
)
except Exception:
logger.exception(
"Failed to report turn execution failure."
)
async def cancel_finalization_task() -> None: async def cancel_finalization_task() -> None:
""" """
Cancel and await the Gateway-owned finalization task. Cancel and await the Gateway-owned finalization task.
@@ -792,7 +822,6 @@ async def audio_socket(websocket: WebSocket) -> None:
continue continue
# --------------------------------------------------------- # ---------------------------------------------------------
# audio.stop # audio.stop
# --------------------------------------------------------- # ---------------------------------------------------------
@@ -805,37 +834,41 @@ async def audio_socket(websocket: WebSocket) -> None:
"STT session has not been started." "STT session has not been started."
) )
if pending_audio_chunk is not None:
raise ValueError(
"audio.stop received while an audio chunk is pending."
)
if agent is None:
raise RuntimeError(
"Agent adapter is not initialized."
)
if tts is None:
raise RuntimeError(
"TTS adapter is not initialized."
)
if pipeline is None: if pipeline is None:
raise RuntimeError( raise RuntimeError(
"Voice pipeline is not initialized." "Voice pipeline is not initialized."
) )
# Finish the STT session normally. if agent is None or tts is None:
await stt.stop_session()
await pipeline.wait_turn_task()
stt_events_task = None
try:
pipeline.complete_turn(request.turn_id)
except (RuntimeError, ValueError) as exc:
raise RuntimeError( raise RuntimeError(
"Turn pipeline did not complete successfully." "Agent or TTS adapter is not initialized."
) from exc )
if not pipeline.is_active_turn(request.turn_id):
raise ValueError(
f"Unexpected turn_id={request.turn_id}."
)
if pending_audio_chunk is not None:
raise ValueError(
"audio.stop received while an audio chunk is pending."
)
if (
finalization_task is not None
and not finalization_task.done()
):
raise RuntimeError(
"Turn finalization is already running."
)
await stt.stop_session()
# Do not await OCI/TTS here.
# Keep the WebSocket receive loop available.
finalization_task = asyncio.create_task(
finalize_turn(request.turn_id)
)
except (ValidationError, RuntimeError, ValueError) as exc: except (ValidationError, RuntimeError, ValueError) as exc:
await send_json( await send_json(
@@ -847,20 +880,10 @@ async def audio_socket(websocket: WebSocket) -> None:
turn_id=payload.get("turn_id"), turn_id=payload.get("turn_id"),
), ),
) )
continue
stt = None
pending_audio_chunk = None
await send_json(
websocket,
TurnEnded(
session_id=session.session_id,
turn_id=request.turn_id,
),
)
continue continue
# --------------------------------------------------------- # ---------------------------------------------------------
# interrupt # interrupt
# --------------------------------------------------------- # ---------------------------------------------------------
@@ -895,8 +918,11 @@ async def audio_socket(websocket: WebSocket) -> None:
# Only cancel resources belonging to the requested active turn. # Only cancel resources belonging to the requested active turn.
if pipeline.is_active_turn(request.turn_id): if pipeline.is_active_turn(request.turn_id):
try: try:
# Cancel and await STT/Agent/TTS execution before async with turn_transition_lock:
# confirming the interruption to the client. interrupting_turns.add(request.turn_id)
await cancel_finalization_task()
await pipeline.cancel_turn_task() await pipeline.cancel_turn_task()
# Abort transcription without requesting a final result. # Abort transcription without requesting a final result.
@@ -923,8 +949,19 @@ async def audio_socket(websocket: WebSocket) -> None:
continue continue
else: else:
# Preserve the existing tolerant behavior for unknown turns. await send_json(
pipeline.interrupt_if_active(request.turn_id) websocket,
error_event(
"INVALID_TURN",
(
f"Turn {request.turn_id} "
"is no longer active."
),
session_id=session.session_id,
turn_id=request.turn_id,
),
)
continue
await send_json( await send_json(
websocket, websocket,
@@ -946,6 +983,12 @@ async def audio_socket(websocket: WebSocket) -> None:
raise RuntimeError( raise RuntimeError(
"Voice pipeline is not initialized." "Voice pipeline is not initialized."
) )
# Prevent normal completion during session shutdown.
async with turn_transition_lock:
if session.active_turn_id is not None:
interrupting_turns.add(session.active_turn_id)
await cancel_finalization_task()
# Cancel and await any running STT/Agent/TTS task. # Cancel and await any running STT/Agent/TTS task.
await pipeline.cancel_turn_task() await pipeline.cancel_turn_task()
@@ -1079,3 +1122,15 @@ async def audio_socket(websocket: WebSocket) -> None:
session.session_id if session else None, session.session_id if session else None,
) )
# Release provider-owned HTTP connections.
if agent is not None:
try:
await close_optional_provider(agent)
except Exception:
logger.exception(
"Failed to close Agent provider. session_id=%s",
session.session_id if session else None,
)
finally:
agent = None

View File

@@ -82,8 +82,12 @@ export class PCMPlayer {
} }
async enqueue(buffer) { async enqueue(buffer) {
const generation = this.generation;
if (this.finished) { if (this.finished) {
throw new Error("Playback has already been finalized."); // Playback was already stopped or finalized.
// Late audio chunks must not terminate the conversation.
return;
} }
const samples = this.decodePCM(buffer); const samples = this.decodePCM(buffer);
@@ -94,8 +98,12 @@ export class PCMPlayer {
await this.context.resume(); await this.context.resume();
if (generation !== this.generation) {
return;
}
if (this.finished) { if (this.finished) {
throw new Error("Playback was finalized during enqueue."); return;
} }
const audioBuffer = this.context.createBuffer( const audioBuffer = this.context.createBuffer(
@@ -116,9 +124,6 @@ export class PCMPlayer {
); );
this.nextStartTime = startAt + audioBuffer.duration; this.nextStartTime = startAt + audioBuffer.duration;
const generation = this.generation;
this.sources.add(source); this.sources.add(source);
if (!this.playing) { if (!this.playing) {

View File

@@ -47,6 +47,22 @@ export class VoiceSessionController {
this.playbackPending = Promise.resolve(); this.playbackPending = Promise.resolve();
this.gatewayTurnEnded = false; this.gatewayTurnEnded = false;
this.playbackGeneration = 0; this.playbackGeneration = 0;
this.bargeInPending = false;
this.bargeInBuffer = [];
this.maxBargeInChunks = 100;
this.bargeInSpeechEnded = false;
this.interruptedTurnId = null;
this.bargeInCaptureStarted = false;
this.bargeInVad = new VoiceActivityDetector({
minSpeechMs: 350,
silenceMs: 800,
threshold: 0.025,
noiseMultiplier: 4,
onSpeechStart: () => this.requestBargeIn(),
onSpeechEnd: () => this.finishBargeInSpeech(),
});
} }
getPlaybackAnalyser() { getPlaybackAnalyser() {
@@ -54,6 +70,16 @@ export class VoiceSessionController {
} }
setState(state) { setState(state) {
console.debug(
`[AgentVoice][STATE] ${this.state} -> ${state}`,
{
turnId: this.turnId,
playerFinished: this.player?.finished,
bargeInPending: this.bargeInPending,
bargeInRecording: this.bargeInRecording,
}
);
this.state = state; this.state = state;
this.onStateChange(state); this.onStateChange(state);
} }
@@ -78,11 +104,16 @@ export class VoiceSessionController {
audioContext, audioContext,
sampleRate: 48000, sampleRate: 48000,
onPlaybackStart: () => { onPlaybackStart: () => {
if (this.state !== "closed") { if (
this.setState("speaking"); this.state === "processing" ||
} this.state === "speaking"
}, ) {
this.setState("speaking");
this.bargeInVad.start();
this.monitorVoice();
}
},
}); });
this.rmsSamples = new Float32Array(analyser.fftSize); this.rmsSamples = new Float32Array(analyser.fftSize);
@@ -162,67 +193,102 @@ export class VoiceSessionController {
} }
} }
startListening() { startListening() {
this.playbackGeneration++; this.playbackGeneration++;
this.gatewayTurnEnded = false; this.gatewayTurnEnded = false;
this.player?.reset(); this.player?.reset();
this.playbackPending = Promise.resolve(); this.playbackPending = Promise.resolve();
this.turnId = null; this.bargeInVad.stop();
this.sequence = 0; this.bargeInPending = false;
this.prebuffer = []; this.bargeInSpeechEnded = false;
this.pending = []; this.interruptedTurnId = null;
this.stopRequested = false; this.bargeInBuffer = [];
this.worklet.port.postMessage({ type: "start" }); this.turnId = null;
this.vad.start(); this.sequence = 0;
this.setState("listening"); this.prebuffer = [];
this.pending = [];
this.stopRequested = false;
this.monitorVoice(); this.worklet.port.postMessage({ type: "start" });
this.vad.start();
this.setState("listening");
this.monitorVoice();
} }
monitorVoice() {
cancelAnimationFrame(this.frameId);
const tick = (now) => { monitorVoice() {
if (this.state !== "listening" && cancelAnimationFrame(this.frameId);
this.state !== "starting" &&
this.state !== "recording") {
return;
}
this.analyser.getFloatTimeDomainData(this.rmsSamples); const tick = (now) => {
const normalListening = [
"listening",
"starting",
"recording"
].includes(this.state);
let power = 0; const bargeInListening = [
for (const sample of this.rmsSamples) { "processing",
power += sample * sample; "speaking",
} "waiting-playback",
"interrupting"
].includes(this.state) || this.bargeInRecording;
const rms = Math.sqrt(power / this.rmsSamples.length); if (!normalListening && !bargeInListening) {
return;
}
if (!this.stopRequested) { this.analyser.getFloatTimeDomainData(this.rmsSamples);
this.vad.process(rms, now);
} let power = 0;
for (const sample of this.rmsSamples) {
power += sample * sample;
}
const rms = Math.sqrt(
power / this.rmsSamples.length
);
// Normal speech detection.
if (
normalListening &&
!this.bargeInRecording &&
!this.stopRequested
) {
this.vad.process(rms, now);
}
// Continue tracking speech initiated during TTS.
if (
bargeInListening &&
this.bargeInVad.active
) {
this.bargeInVad.process(rms, now);
}
this.frameId = requestAnimationFrame(tick);
};
this.frameId = requestAnimationFrame(tick); this.frameId = requestAnimationFrame(tick);
};
this.frameId = requestAnimationFrame(tick);
}
beginTurn() {
if (this.state !== "listening") return;
this.pending = [...this.prebuffer];
this.setState("starting");
try {
this.send({ type: "audio.start" });
} catch (error) {
this.fail(error);
} }
}
beginTurn() {
if (this.state !== "listening") return;
this.pending = [...this.prebuffer];
this.setState("starting");
try {
this.send({ type: "audio.start" });
} catch (error) {
this.fail(error);
}
}
handleWorkletMessage(data) { handleWorkletMessage(data) {
if (data?.type === "pcm") { if (data?.type === "pcm") {
@@ -242,6 +308,21 @@ export class VoiceSessionController {
return; return;
} }
if (
this.state === "processing" ||
this.state === "speaking" ||
this.state === "waiting-playback" ||
this.state === "interrupting"
) {
this.bargeInBuffer.push(buffer);
if (this.bargeInBuffer.length > this.maxBargeInChunks) {
this.bargeInBuffer.shift();
}
return;
}
if (this.state === "starting") { if (this.state === "starting") {
this.pending.push(buffer); this.pending.push(buffer);
@@ -302,31 +383,80 @@ export class VoiceSessionController {
}); });
this.setState("processing"); this.setState("processing");
cancelAnimationFrame(this.frameId);
// Resume local PCM capture for a possible interruption.
// Audio will remain buffered until a new turn is accepted.
this.worklet.port.postMessage({ type: "start" });
// The microphone remains available for barge-in.
// Detection starts when TTS playback begins.
this.monitorVoice();
} }
handleAudioChunk(buffer) {
handleAudioChunk(buffer) {
if ( if (
this.state === "closed" || this.state === "closed" ||
this.turnId === null || this.state === "interrupting" ||
this.gatewayTurnEnded this.bargeInPending ||
this.turnId === null ||
this.gatewayTurnEnded
) { ) {
return; return;
} }
console.debug("[AgentVoice][PCM recebido]", {
bytes: buffer.byteLength,
turnId: this.turnId,
state: this.state,
generation: this.playbackGeneration,
finished: this.player?.finished,
gatewayTurnEnded: this.gatewayTurnEnded,
});
const generation = this.playbackGeneration; const generation = this.playbackGeneration;
this.playbackPending = this.playbackPending this.playbackPending = this.playbackPending
.then(async () => { .then(async () => {
if (generation !== this.playbackGeneration) return; if (generation !== this.playbackGeneration) {
await this.player.enqueue(buffer); return;
})
.catch((error) => {
if (generation === this.playbackGeneration) {
this.fail(error);
} }
});
} if (
this.state === "interrupting" ||
this.bargeInPending
) {
return;
}
console.debug("[AgentVoice][PCM enqueue]", {
turnId: this.turnId,
state: this.state,
generation,
currentGeneration: this.playbackGeneration,
finished: this.player?.finished,
});
await this.player.enqueue(buffer);
})
.catch((error) => {
// An obsolete playback must never terminate the session.
if (generation !== this.playbackGeneration) {
return;
}
if (
this.state === "interrupting" ||
this.bargeInPending
) {
return;
}
this.fail(error);
});
}
async finishPlaybackAndResume() { async finishPlaybackAndResume() {
const generation = this.playbackGeneration; const generation = this.playbackGeneration;
@@ -353,6 +483,66 @@ export class VoiceSessionController {
} }
requestBargeIn() {
if (this.bargeInPending) return;
if (
this.state !== "speaking" &&
this.state !== "waiting-playback"
) {
return;
}
if (this.turnId === null) return;
const interruptedTurnId = this.turnId;
this.bargeInPending = true;
this.bargeInSpeechEnded = false;
this.bargeInCaptureStarted = true;
this.interruptedTurnId = interruptedTurnId;
// Stop local playback immediately.
this.playbackGeneration++;
this.player?.stop();
this.playbackPending = Promise.resolve();
// Prevent normal playback completion from resuming listening.
this.gatewayTurnEnded = false;
// The normal VAD must not start a competing turn.
this.vad.stop();
this.setState("interrupting");
try {
this.send({
type: "interrupt",
turn_id: interruptedTurnId,
});
} catch (error) {
this.fail(error);
}
}
finishBargeInSpeech() {
if (!this.bargeInPending && !this.bargeInRecording) {
return;
}
this.bargeInSpeechEnded = true;
if (
this.bargeInRecording &&
this.state === "recording"
) {
this.bargeInRecording = false;
this.bargeInVad.stop();
this.stopRequested = true;
this.flushTurn();
}
}
handleGatewayEvent(event) { handleGatewayEvent(event) {
switch (event.type) { switch (event.type) {
@@ -360,9 +550,39 @@ export class VoiceSessionController {
this.startListening(); this.startListening();
break; break;
// Debug logging for all events received from the Gateway.
console.debug("[AgentVoice][Gateway]", {
type: event.type,
turnId: event.turn_id,
currentTurnId: this.turnId,
state: this.state,
playbackGeneration: this.playbackGeneration,
playerFinished: this.player?.finished,
playerPlaying: this.player?.playing,
bargeInPending: this.bargeInPending,
bargeInRecording: this.bargeInRecording,
text: event.type === "stt.final"
? event.text
: event.type === "agent.response"
? event.text
: undefined,
error: event.type === "error"
? { code: event.code, message: event.message }
: undefined,
});
// Debug logging for all events received from the Gateway.
case "turn.started": case "turn.started":
if (this.state !== "starting") return; if (this.state !== "starting") return;
// Prepare playback for the newly accepted turn.
// The previous TTS may have been stopped by barge-in.
this.playbackGeneration++;
this.player?.reset();
this.playbackPending = Promise.resolve();
this.gatewayTurnEnded = false;
this.turnId = event.turn_id; this.turnId = event.turn_id;
this.sequence = 0; this.sequence = 0;
@@ -374,6 +594,25 @@ export class VoiceSessionController {
this.pending = []; this.pending = [];
if (this.bargeInPending) {
this.bargeInPending = false;
this.bargeInCaptureStarted = false;
if (this.bargeInSpeechEnded) {
// User finished speaking before turn.started.
this.bargeInRecording = false;
this.bargeInVad.stop();
this.stopRequested = true;
} else {
// User is still speaking.
// Preserve the active barge-in VAD state.
this.bargeInRecording = true;
this.stopRequested = false;
}
this.bargeInSpeechEnded = false;
}
if (this.stopRequested) { if (this.stopRequested) {
this.flushTurn(); this.flushTurn();
} }
@@ -399,19 +638,71 @@ export class VoiceSessionController {
break; break;
case "turn.ended": case "turn.ended":
if (event.turn_id === this.turnId) { if (
this.gatewayTurnEnded = true; event.turn_id === this.turnId &&
this.setState("waiting-playback"); !this.bargeInPending
) {
this.gatewayTurnEnded = true;
this.setState("waiting-playback");
void this.finishPlaybackAndResume(); void this.finishPlaybackAndResume();
} }
break; break;
case "interrupted": case "interrupted":
this.close(); if (
!this.bargeInPending ||
event.turn_id !== this.interruptedTurnId
) {
return;
}
this.interruptedTurnId = null;
this.turnId = null;
this.sequence = 0;
this.pending = [...this.bargeInBuffer];
this.bargeInBuffer = [];
this.stopRequested = this.bargeInSpeechEnded;
this.setState("starting");
try {
this.send({ type: "audio.start" });
} catch (error) {
this.fail(error);
}
break; break;
case "error": case "error":
if (
event.code === "INVALID_TURN" &&
this.bargeInPending &&
event.turn_id === this.interruptedTurnId
) {
// The old turn completed before interrupt was processed.
// Its semantic state is already released by the Gateway.
this.interruptedTurnId = null;
this.turnId = null;
this.sequence = 0;
this.pending = [...this.bargeInBuffer];
this.bargeInBuffer = [];
this.stopRequested = this.bargeInSpeechEnded;
this.gatewayTurnEnded = false;
this.setState("starting");
try {
this.send({ type: "audio.start" });
} catch (error) {
this.fail(error);
}
break;
}
this.fail( this.fail(
new Error(`${event.code}: ${event.message}`) new Error(`${event.code}: ${event.message}`)
); );
@@ -435,6 +726,14 @@ export class VoiceSessionController {
this.player?.stop(); this.player?.stop();
this.vad.stop(); this.vad.stop();
this.bargeInVad.stop();
this.bargeInPending = false;
this.bargeInSpeechEnded = false;
this.bargeInCaptureStarted = false;
this.interruptedTurnId = null;
this.bargeInBuffer = [];
cancelAnimationFrame(this.frameId); cancelAnimationFrame(this.frameId);
if (this.worklet) { if (this.worklet) {

View File

@@ -202,3 +202,24 @@ test("exposes playback analyser", () => {
player.analyser player.analyser
); );
}); });
test("late PCM after stop is ignored", async () => {
const context = new FakeAudioContext();
const player = new PCMPlayer({
audioContext: context,
sampleRate: 48000,
});
player.stop();
const pcm = new ArrayBuffer(8);
await assert.doesNotReject(
player.enqueue(pcm)
);
assert.equal(player.playing, false);
assert.equal(player.sources.size, 0);
});

View File

@@ -29,6 +29,8 @@ class FakeWebSocket {
globalThis.WebSocket = FakeWebSocket; globalThis.WebSocket = FakeWebSocket;
globalThis.cancelAnimationFrame = () => {}; globalThis.cancelAnimationFrame = () => {};
globalThis.requestAnimationFrame = () => 1;
afterEach(() => { afterEach(() => {
// Restore only state modified by individual tests. // Restore only state modified by individual tests.
@@ -145,27 +147,31 @@ test("speech end requests worklet flush", () => {
assert.equal(controller.ws.sent.length, 0); assert.equal(controller.ws.sent.length, 0);
}); });
test("audio.stop is sent after flushed", () => { test("audio.stop is sent after flushed", () => {
const { controller } = makeController(); const { controller } = makeController();
controller.turnId = 1; controller.turnId = 1;
controller.setState("flushing"); controller.setState("flushing");
controller.handlePCM(new ArrayBuffer(4));
controller.handleFlushed(); controller.handleFlushed();
const messages = controller.ws.sent
.filter((message) => typeof message === "string")
.map((message) => JSON.parse(message));
assert.ok(
messages.some(
(message) =>
message.type === "audio.stop" &&
message.turn_id === 1
)
);
assert.equal(controller.state, "processing"); assert.equal(controller.state, "processing");
const lastMessage = JSON.parse(controller.ws.sent.at(-1));
assert.deepEqual(lastMessage, {
type: "audio.stop",
turn_id: 1,
});
assert.equal(controller.ws.sent.length, 3);
}); });
test("speech ending before turn.started is handled", () => { test("speech ending before turn.started is handled", () => {
const { controller, workletMessages } = makeController(); const { controller, workletMessages } = makeController();
@@ -337,3 +343,393 @@ test("binary audio is forwarded to PCMPlayer", async () => {
assert.deepEqual(received, [pcm]); assert.deepEqual(received, [pcm]);
}); });
test("barge-in stops playback and starts a new turn", () => {
const { controller } = makeController();
let playbackStopped = false;
controller.player = {
stop() {
playbackStopped = true;
},
reset() {},
};
controller.turnId = 1;
controller.setState("speaking");
// Simulate user speech detected during TTS.
controller.requestBargeIn();
assert.equal(playbackStopped, true);
assert.equal(controller.state, "interrupting");
assert.equal(controller.bargeInPending, true);
assert.deepEqual(
JSON.parse(controller.ws.sent[0]),
{
type: "interrupt",
turn_id: 1,
}
);
// Gateway confirms the previous turn was cancelled.
controller.handleGatewayEvent({
type: "interrupted",
turn_id: 1,
});
assert.equal(controller.state, "starting");
assert.equal(controller.ws.closed, false);
assert.deepEqual(
JSON.parse(controller.ws.sent[1]),
{
type: "audio.start",
}
);
// Gateway accepts the new turn.
controller.handleGatewayEvent({
type: "turn.started",
turn_id: 2,
});
assert.equal(controller.turnId, 2);
assert.equal(controller.state, "recording");
assert.equal(controller.bargeInPending, false);
});
test("barge-in preserves PCM until new turn starts", () => {
const { controller } = makeController();
controller.player = {
stop() {},
reset() {},
};
controller.turnId = 1;
controller.setState("speaking");
const pcm1 = new ArrayBuffer(4);
const pcm2 = new ArrayBuffer(6);
const pcm3 = new ArrayBuffer(8);
// Audio captured before speech confirmation.
controller.handlePCM(pcm1);
controller.requestBargeIn();
// Audio captured while interruption is pending.
controller.handlePCM(pcm2);
assert.equal(controller.ws.sent.length, 1);
assert.equal(controller.bargeInBuffer.length, 2);
controller.handleGatewayEvent({
type: "interrupted",
turn_id: 1,
});
// Still no PCM sent to the old turn.
assert.equal(controller.state, "starting");
assert.equal(controller.ws.sent.length, 2);
// Additional speech arrives before turn.started.
controller.handlePCM(pcm3);
controller.handleGatewayEvent({
type: "turn.started",
turn_id: 2,
});
const sent = controller.ws.sent;
assert.deepEqual(
JSON.parse(sent[0]),
{ type: "interrupt", turn_id: 1 }
);
assert.deepEqual(
JSON.parse(sent[1]),
{ type: "audio.start" }
);
assert.deepEqual(
JSON.parse(sent[2]),
{
type: "audio.chunk",
turn_id: 2,
sequence_number: 0,
}
);
assert.equal(sent[3], pcm1);
assert.deepEqual(
JSON.parse(sent[4]),
{
type: "audio.chunk",
turn_id: 2,
sequence_number: 1,
}
);
assert.equal(sent[5], pcm2);
assert.deepEqual(
JSON.parse(sent[6]),
{
type: "audio.chunk",
turn_id: 2,
sequence_number: 2,
}
);
assert.equal(sent[7], pcm3);
assert.equal(controller.sequence, 3);
assert.equal(controller.state, "recording");
});
test("barge-in finishes speech before new turn starts", () => {
const { controller, workletMessages } = makeController();
controller.player = {
stop() {},
reset() {},
};
controller.turnId = 1;
controller.setState("speaking");
const pcm = new ArrayBuffer(8);
controller.handlePCM(pcm);
controller.requestBargeIn();
// User finishes speaking before the Gateway responds.
controller.finishBargeInSpeech();
assert.equal(controller.bargeInSpeechEnded, true);
controller.handleGatewayEvent({
type: "interrupted",
turn_id: 1,
});
assert.equal(controller.state, "starting");
controller.handleGatewayEvent({
type: "turn.started",
turn_id: 2,
});
// Buffered audio must be transmitted before flushing.
assert.equal(controller.sequence, 1);
assert.deepEqual(
JSON.parse(controller.ws.sent[2]),
{
type: "audio.chunk",
turn_id: 2,
sequence_number: 0,
}
);
assert.equal(controller.ws.sent[3], pcm);
assert.equal(controller.state, "flushing");
assert.deepEqual(
workletMessages.at(-1),
{ type: "flush" }
);
});
test("barge-in continues recording until speech ends", () => {
const { controller, workletMessages } = makeController();
controller.player = {
stop() {},
reset() {},
};
controller.turnId = 1;
controller.setState("speaking");
controller.bargeInVad.start();
controller.requestBargeIn();
controller.handleGatewayEvent({
type: "interrupted",
turn_id: 1,
});
controller.handleGatewayEvent({
type: "turn.started",
turn_id: 2,
});
assert.equal(controller.state, "recording");
assert.equal(controller.bargeInRecording, true);
assert.equal(controller.bargeInVad.active, true);
// Speech continues after the new turn starts.
const pcm = new ArrayBuffer(8);
controller.handlePCM(pcm);
assert.equal(controller.sequence, 1);
// Barge-in VAD detects the end of speech.
controller.finishBargeInSpeech();
assert.equal(controller.bargeInRecording, false);
assert.equal(controller.state, "flushing");
assert.deepEqual(
workletMessages.at(-1),
{ type: "flush" }
);
// Worklet confirms that remaining PCM was emitted.
controller.handleFlushed();
assert.equal(controller.state, "processing");
const messages = controller.ws.sent
.filter((message) => typeof message === "string")
.map((message) => JSON.parse(message));
assert.ok(
messages.some(
(message) =>
message.type === "audio.stop" &&
message.turn_id === 2
)
);
});
test("late INVALID_TURN recovers barge-in session", () => {
const { controller } = makeController();
controller.player = {
stop() {},
reset() {},
};
controller.turnId = 1;
controller.setState("speaking");
controller.requestBargeIn();
assert.equal(controller.state, "interrupting");
controller.handleGatewayEvent({
type: "error",
code: "INVALID_TURN",
message: "Turn 1 is no longer active.",
turn_id: 1,
});
assert.equal(controller.state, "starting");
assert.equal(controller.ws.closed, false);
const messages = controller.ws.sent
.filter((message) => typeof message === "string")
.map((message) => JSON.parse(message));
assert.deepEqual(
messages.map((message) => message.type),
["interrupt", "audio.start"]
);
controller.handleGatewayEvent({
type: "turn.started",
turn_id: 2,
});
assert.equal(controller.state, "recording");
assert.equal(controller.turnId, 2);
});
test("late TTS chunk during barge-in does not close session", () => {
const { controller } = makeController();
let stopped = false;
controller.player = {
stop() {
stopped = true;
},
reset() {},
};
controller.turnId = 1;
controller.setState("speaking");
controller.requestBargeIn();
assert.equal(stopped, true);
assert.equal(controller.state, "interrupting");
// A late PCM chunk from the interrupted TTS.
controller.handleAudioChunk(new ArrayBuffer(8));
// The WebSocket session must remain alive.
assert.equal(controller.ws.closed, false);
assert.equal(controller.state, "interrupting");
assert.equal(controller.bargeInPending, true);
});
test("barge-in resets player before new TTS response", () => {
const { controller } = makeController();
let finished = false;
let resetCount = 0;
controller.player = {
stop() {
finished = true;
},
reset() {
finished = false;
resetCount++;
},
};
controller.turnId = 1;
controller.setState("speaking");
controller.requestBargeIn();
assert.equal(finished, true);
controller.handleGatewayEvent({
type: "interrupted",
turn_id: 1,
});
assert.equal(controller.state, "starting");
controller.handleGatewayEvent({
type: "turn.started",
turn_id: 2,
});
assert.equal(controller.state, "recording");
assert.equal(finished, false);
assert.equal(resetCount, 1);
});

343
tests/test_deep_agent.py Normal file
View File

@@ -0,0 +1,343 @@
import pytest
from unittest.mock import AsyncMock, MagicMock
from voice_assistant.agent.adapter import AgentAdapter
def test_deep_agent_implements_port():
from voice_assistant.agent.deep_agent import DeepAgentsAdapter
assert issubclass(DeepAgentsAdapter, AgentAdapter)
assert callable(getattr(DeepAgentsAdapter, "stream", None))
assert callable(getattr(DeepAgentsAdapter, "aclose", None))
def test_deep_agent_session_config():
from voice_assistant.agent.deep_agent import DeepAgentsAdapter
adapter = object.__new__(DeepAgentsAdapter)
assert adapter._config("session-a") == {
"configurable": {
"thread_id": "session-a",
}
}
assert adapter._config("session-b") == {
"configurable": {
"thread_id": "session-b",
}
}
with pytest.raises(ValueError):
adapter._config("")
def test_deep_agent_cleanup_is_idempotent():
import asyncio
from voice_assistant.agent.deep_agent import DeepAgentsAdapter
adapter = object.__new__(DeepAgentsAdapter)
adapter.model = MagicMock()
adapter.model.aclose = AsyncMock()
adapter._closed = False
async def scenario():
await adapter.aclose()
await adapter.aclose()
asyncio.run(scenario())
adapter.model.aclose.assert_awaited_once()
assert adapter._closed is True
def test_deep_agent_stream_can_be_cancelled():
import asyncio
from voice_assistant.agent.deep_agent import DeepAgentsAdapter
async def scenario():
adapter = object.__new__(DeepAgentsAdapter)
adapter._closed = False
started = asyncio.Event()
cancelled = asyncio.Event()
class FakeAgent:
async def astream(self, *args, **kwargs):
started.set()
try:
yield (
FakeMessage("Primeiro fragmento"),
{"langgraph_node": "model"},
)
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelled.set()
raise
class FakeMessage:
def __init__(self, content):
self.content = content
self.tool_call_chunks = []
adapter.agent = FakeAgent()
async def consume():
async for _ in adapter.stream(
session_id="session-cancel",
turn_id=1,
text="Teste de cancelamento",
):
pass
task = asyncio.create_task(consume())
await asyncio.wait_for(started.wait(), timeout=2.0)
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
assert cancelled.is_set()
assert task.done()
asyncio.run(scenario())
def test_deep_agent_isolates_session_history():
import asyncio
from deepagents import create_deep_agent
from langchain_core.messages import HumanMessage
from langchain_core.language_models import FakeListChatModel
from langgraph.checkpoint.memory import MemorySaver
async def scenario():
checkpointer = MemorySaver()
class ToolCompatibleFakeChatModel(FakeListChatModel):
def bind_tools(self, tools, *, tool_choice=None, **kwargs):
return self
model = ToolCompatibleFakeChatModel(
responses=[
"Resposta A1",
"Resposta B1",
"Resposta A2",
]
)
agent = create_deep_agent(
model=model,
tools=[],
system_prompt="Responda de forma breve.",
checkpointer=checkpointer,
)
async def ask(session_id, text):
return await agent.ainvoke(
{
"messages": [
HumanMessage(content=text)
]
},
config={
"configurable": {
"thread_id": session_id,
}
},
)
await ask("session-a", "Destino: Japão")
await ask("session-b", "Destino: Canadá")
await ask("session-a", "Qual era meu destino?")
def history(session_id):
snapshot = agent.get_state({
"configurable": {
"thread_id": session_id,
}
})
return snapshot.values.get("messages", [])
history_a = history("session-a")
history_b = history("session-b")
texts_a = [
message.content
for message in history_a
if isinstance(message, HumanMessage)
]
texts_b = [
message.content
for message in history_b
if isinstance(message, HumanMessage)
]
assert texts_a == [
"Destino: Japão",
"Qual era meu destino?",
]
assert texts_b == [
"Destino: Canadá",
]
assert all(
"Canadá" not in text
for text in texts_a
)
assert all(
"Japão" not in text
for text in texts_b
)
asyncio.run(scenario())
def test_deep_agent_checkpoint_survives_cancellation():
import asyncio
from deepagents import create_deep_agent
from langchain_core.language_models import FakeListChatModel
from langchain_core.messages import HumanMessage
from langgraph.checkpoint.memory import MemorySaver
class CancellableFakeModel(FakeListChatModel):
def bind_tools(self, tools, *, tool_choice=None, **kwargs):
return self
async def _agenerate(
self,
messages,
stop=None,
run_manager=None,
**kwargs,
):
last_user_message = next(
(
message
for message in reversed(messages)
if isinstance(message, HumanMessage)
),
None,
)
if (
last_user_message is not None
and last_user_message.content == "Interromper esta resposta"
):
started.set()
await asyncio.Event().wait()
return await super()._agenerate(
messages,
stop=stop,
run_manager=run_manager,
**kwargs,
)
async def scenario():
nonlocal started
started = asyncio.Event()
model = CancellableFakeModel(
responses=[
"Contexto inicial confirmado.",
"Contexto preservado.",
]
)
agent = create_deep_agent(
model=model,
tools=[],
system_prompt="Responda brevemente.",
checkpointer=MemorySaver(),
)
config = {
"configurable": {
"thread_id": "session-cancellation-test",
}
}
async def ask(text):
return await agent.ainvoke(
{
"messages": [
HumanMessage(content=text)
]
},
config=config,
)
# First turn completes successfully.
await ask("Meu destino é Japão")
# Second turn begins but never completes.
task = asyncio.create_task(
ask("Interromper esta resposta")
)
await asyncio.wait_for(
started.wait(),
timeout=2.0,
)
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
assert task.done()
assert task.cancelled()
# The same thread must remain usable.
await asyncio.wait_for(
ask("Qual era meu destino?"),
timeout=5.0,
)
snapshot = await agent.aget_state(config)
messages = snapshot.values.get("messages", [])
human_messages = [
message.content
for message in messages
if isinstance(message, HumanMessage)
]
assert "Meu destino é Japão" in human_messages
assert "Qual era meu destino?" in human_messages
# An interrupted execution must not be recorded
# as a completed assistant response.
assert not any(
getattr(message, "content", None)
== "Resposta interrompida concluída."
for message in messages
)
started = None
asyncio.run(scenario())

View File

@@ -348,6 +348,142 @@ def test_websocket_session_end():
assert session_ended["session_id"] == session_id assert session_ended["session_id"] == session_id
def test_websocket_session_end_closes_agent_provider():
"""
Explicit session.end must close the Agent provider.
"""
agent = MockAgentAdapter()
agent.aclose = AsyncMock()
with patch.object(
main,
"create_agent_adapter",
return_value=agent,
):
with client.websocket_connect("/ws/audio") as websocket:
websocket.send_json({
"type": "session.start",
"audio": {
"encoding": AUDIO_ENCODING,
"sample_rate": AUDIO_SAMPLE_RATE,
"channels": AUDIO_CHANNELS,
},
})
ready = websocket.receive_json()
assert ready["type"] == "session.ready"
websocket.send_json({
"type": "session.end",
})
ended = websocket.receive_json()
assert ended["type"] == "session.ended"
agent.aclose.assert_awaited_once()
def test_websocket_disconnect_closes_agent_provider():
"""
Unexpected WebSocket disconnect must close the Agent provider.
"""
agent = MockAgentAdapter()
agent.aclose = AsyncMock()
with patch.object(
main,
"create_agent_adapter",
return_value=agent,
):
with client.websocket_connect("/ws/audio") as websocket:
websocket.send_json({
"type": "session.start",
"audio": {
"encoding": AUDIO_ENCODING,
"sample_rate": AUDIO_SAMPLE_RATE,
"channels": AUDIO_CHANNELS,
},
})
ready = websocket.receive_json()
assert ready["type"] == "session.ready"
# Leave without sending session.end.
# TestClient closes the WebSocket context.
agent.aclose.assert_awaited_once()
def test_websocket_interrupt_preserves_agent_provider():
"""
Interrupt must cancel the turn without closing the Agent provider.
The provider is closed only when the WebSocket session ends.
"""
created_stt.clear()
agent = MockAgentAdapter()
agent.aclose = AsyncMock()
with patch.object(
main,
"create_agent_adapter",
return_value=agent,
):
with client.websocket_connect("/ws/audio") as websocket:
websocket.send_json({
"type": "session.start",
"audio": {
"encoding": AUDIO_ENCODING,
"sample_rate": AUDIO_SAMPLE_RATE,
"channels": AUDIO_CHANNELS,
},
})
ready = websocket.receive_json()
assert ready["type"] == "session.ready"
websocket.send_json({
"type": "audio.start",
})
started = websocket.receive_json()
assert started["type"] == "turn.started"
websocket.send_json({
"type": "interrupt",
"turn_id": started["turn_id"],
})
interrupted = websocket.receive_json()
assert interrupted["type"] == "interrupted"
# Interrupt must not close the provider.
agent.aclose.assert_not_awaited()
# The same WebSocket must accept a new turn.
websocket.send_json({
"type": "audio.start",
})
next_turn = websocket.receive_json()
assert next_turn["type"] == "turn.started"
assert next_turn["turn_id"] != started["turn_id"]
# Provider must remain open.
agent.aclose.assert_not_awaited()
websocket.send_json({
"type": "session.end",
})
ended = websocket.receive_json()
assert ended["type"] == "session.ended"
agent.aclose.assert_awaited_once()
def test_websocket_forwards_stt_events(): def test_websocket_forwards_stt_events():
created_stt.clear() created_stt.clear()
created_turn_managers.clear() created_turn_managers.clear()
@@ -1800,3 +1936,477 @@ def test_websocket_streaming_agent_and_tts():
turn_ended = websocket.receive_json() turn_ended = websocket.receive_json()
assert turn_ended["type"] == "turn.ended" assert turn_ended["type"] == "turn.ended"
def test_websocket_interrupt_during_streaming_tts():
"""
Verify that interrupt cancels an active streaming TTS turn
while normal turn finalization is pending.
"""
created_stt.clear()
tts_started = threading.Event()
tts_cancelled = threading.Event()
class SlowStreamingAgent:
async def stream(self, session_id, turn_id, text):
yield "Olá, esta é uma resposta longa."
class SlowStreamingTTS:
async def synthesize(self, text):
try:
tts_started.set()
yield b"\x01\x02\x03\x04"
# Keep synthesis active until cancellation.
await asyncio.Event().wait()
except asyncio.CancelledError:
tts_cancelled.set()
raise
with patch.object(
main,
"create_agent_adapter",
return_value=SlowStreamingAgent(),
):
with patch.object(
main,
"create_tts_adapter",
return_value=SlowStreamingTTS(),
):
with client.websocket_connect("/ws/audio") as websocket:
websocket.send_json({
"type": "session.start",
"audio": {
"encoding": AUDIO_ENCODING,
"sample_rate": AUDIO_SAMPLE_RATE,
"channels": AUDIO_CHANNELS,
},
})
ready = websocket.receive_json()
assert ready["type"] == "session.ready"
websocket.send_json({"type": "audio.start"})
started = websocket.receive_json()
assert started["type"] == "turn.started"
turn_id = started["turn_id"]
stt = created_stt[-1]
stt.emit_final_now("Olá")
# Wait until the Gateway delivers the first TTS audio chunk.
first_audio_received = False
for _ in range(20):
message = websocket.receive()
if message.get("bytes") is not None:
assert message["bytes"] == b"\x01\x02\x03\x04"
first_audio_received = True
break
import json
event = json.loads(message["text"])
if event["type"] == "error":
raise AssertionError(
f"Gateway returned error: {event}"
)
assert first_audio_received
assert tts_started.is_set()
websocket.send_json({
"type": "audio.stop",
"turn_id": turn_id,
})
websocket.send_json({
"type": "interrupt",
"turn_id": turn_id,
})
# Streaming events may precede interruption.
for _ in range(20):
message = websocket.receive()
if message.get("bytes") is not None:
continue
import json
event = json.loads(message["text"])
if event["type"] == "interrupted":
assert event["turn_id"] == turn_id
break
if event["type"] == "error":
raise AssertionError(
f"Gateway returned error: {event}"
)
else:
raise AssertionError(
"Gateway did not confirm interruption."
)
assert tts_cancelled.wait(timeout=2.0)
# A new turn must be accepted in the same session.
websocket.send_json({"type": "audio.start"})
next_turn = websocket.receive_json()
assert next_turn["type"] == "turn.started"
assert next_turn["turn_id"] != turn_id
websocket.send_json({
"type": "interrupt",
"turn_id": next_turn["turn_id"],
})
interrupted = websocket.receive_json()
assert interrupted["type"] == "interrupted"
def test_websocket_session_end_during_streaming_tts():
"""
Verify session.end cancels an active streaming TTS turn.
The Gateway must cancel provider execution before
acknowledging session closure.
"""
import json
created_stt.clear()
tts_cancelled = threading.Event()
class SlowStreamingAgent:
async def stream(self, session_id, turn_id, text):
yield "Esta é uma resposta longa."
class SlowStreamingTTS:
async def synthesize(self, text):
try:
yield b"\x01\x02\x03\x04"
# Keep synthesis running until cancelled.
await asyncio.Event().wait()
except asyncio.CancelledError:
tts_cancelled.set()
raise
with patch.object(
main,
"create_agent_adapter",
return_value=SlowStreamingAgent(),
):
with patch.object(
main,
"create_tts_adapter",
return_value=SlowStreamingTTS(),
):
with client.websocket_connect("/ws/audio") as websocket:
websocket.send_json({
"type": "session.start",
"audio": {
"encoding": AUDIO_ENCODING,
"sample_rate": AUDIO_SAMPLE_RATE,
"channels": AUDIO_CHANNELS,
},
})
ready = websocket.receive_json()
assert ready["type"] == "session.ready"
websocket.send_json({"type": "audio.start"})
started = websocket.receive_json()
assert started["type"] == "turn.started"
turn_id = started["turn_id"]
stt = created_stt[-1]
stt.emit_final_now("Olá")
# Wait until TTS audio reaches the client.
first_audio_received = False
for _ in range(20):
message = websocket.receive()
if message.get("bytes") is not None:
assert message["bytes"] == b"\x01\x02\x03\x04"
first_audio_received = True
break
event = json.loads(message["text"])
if event["type"] == "error":
raise AssertionError(
f"Gateway returned error: {event}"
)
assert first_audio_received
websocket.send_json({
"type": "audio.stop",
"turn_id": turn_id,
})
websocket.send_json({
"type": "session.end",
})
# Streaming events may already be queued.
for _ in range(20):
message = websocket.receive()
if message.get("bytes") is not None:
continue
event = json.loads(message["text"])
if event["type"] == "session.ended":
assert event["session_id"] == ready["session_id"]
break
if event["type"] == "turn.ended":
raise AssertionError(
"Turn completed normally during session shutdown."
)
if event["type"] == "error":
raise AssertionError(
f"Gateway returned error: {event}"
)
else:
raise AssertionError(
"Gateway did not confirm session closure."
)
assert tts_cancelled.wait(timeout=2.0)
def test_websocket_interrupt_after_turn_completed():
"""
Verify behavior when interrupt arrives after turn.ended.
A completed turn must not be cancelled again.
"""
created_stt.clear()
with client.websocket_connect("/ws/audio") as websocket:
websocket.send_json({
"type": "session.start",
"audio": {
"encoding": AUDIO_ENCODING,
"sample_rate": AUDIO_SAMPLE_RATE,
"channels": AUDIO_CHANNELS,
},
})
ready = websocket.receive_json()
assert ready["type"] == "session.ready"
websocket.send_json({"type": "audio.start"})
started = websocket.receive_json()
assert started["type"] == "turn.started"
turn_id = started["turn_id"]
stt = created_stt[-1]
stt.emit_final_now("Olá")
# Consume events until the first turn finishes.
for _ in range(30):
message = websocket.receive()
if message.get("bytes") is not None:
continue
import json
event = json.loads(message["text"])
if event["type"] == "error":
raise AssertionError(
f"Gateway returned error: {event}"
)
if event["type"] == "stt.final":
websocket.send_json({
"type": "audio.stop",
"turn_id": turn_id,
})
if event["type"] == "turn.ended":
break
else:
raise AssertionError(
"Turn did not complete normally."
)
# Interrupt the already completed turn.
websocket.send_json({
"type": "interrupt",
"turn_id": turn_id,
})
response = websocket.receive_json()
# Existing Gateway behavior may acknowledge stale IDs.
# Capture the current contract before changing it.
assert response["type"] == "error"
assert response["code"] == "INVALID_TURN"
assert response["turn_id"] == turn_id
def test_websocket_interrupt_wins_pending_tts_completion():
"""
Verify interruption wins while TTS completion is pending.
The interrupted turn must not emit turn.ended,
and a new turn must be accepted.
"""
import json
created_stt.clear()
tts_cancelled = threading.Event()
class ControlledAgent:
async def stream(self, session_id, turn_id, text):
yield "Esta resposta será interrompida."
class ControlledTTS:
async def synthesize(self, text):
try:
yield b"\x01\x02\x03\x04"
# Completion remains pending until cancellation.
await asyncio.Event().wait()
except asyncio.CancelledError:
tts_cancelled.set()
raise
with patch.object(
main,
"create_agent_adapter",
return_value=ControlledAgent(),
):
with patch.object(
main,
"create_tts_adapter",
return_value=ControlledTTS(),
):
with client.websocket_connect("/ws/audio") as websocket:
websocket.send_json({
"type": "session.start",
"audio": {
"encoding": AUDIO_ENCODING,
"sample_rate": AUDIO_SAMPLE_RATE,
"channels": AUDIO_CHANNELS,
},
})
ready = websocket.receive_json()
assert ready["type"] == "session.ready"
websocket.send_json({
"type": "audio.start",
})
started = websocket.receive_json()
assert started["type"] == "turn.started"
turn_id = started["turn_id"]
stt = created_stt[-1]
stt.emit_final_now("Olá")
# Observe the first PCM chunk.
for _ in range(20):
message = websocket.receive()
if message.get("bytes") is not None:
assert message["bytes"] == b"\x01\x02\x03\x04"
break
event = json.loads(message["text"])
if event["type"] == "error":
raise AssertionError(event)
else:
raise AssertionError(
"TTS did not produce its first chunk."
)
websocket.send_json({
"type": "audio.stop",
"turn_id": turn_id,
})
websocket.send_json({
"type": "interrupt",
"turn_id": turn_id,
})
terminal_events = []
for _ in range(20):
message = websocket.receive()
if message.get("bytes") is not None:
continue
event = json.loads(message["text"])
if event["type"] in {
"turn.ended",
"interrupted",
}:
terminal_events.append(event["type"])
if event["type"] == "error":
raise AssertionError(event)
if event["type"] == "interrupted":
break
else:
raise AssertionError(
"Interruption was not acknowledged."
)
assert terminal_events == ["interrupted"]
assert tts_cancelled.wait(timeout=2.0)
# Confirm that the session accepts another turn.
websocket.send_json({
"type": "audio.start",
})
next_turn = websocket.receive_json()
assert next_turn["type"] == "turn.started"
assert next_turn["turn_id"] != turn_id
websocket.send_json({
"type": "interrupt",
"turn_id": next_turn["turn_id"],
})
response = websocket.receive_json()
assert response["type"] == "interrupted"
assert response["turn_id"] == next_turn["turn_id"]

View File

@@ -0,0 +1,64 @@
"""
Smoke test for DeepAgentsAdapter streaming and context.
"""
import asyncio
import os
from contextlib import aclosing
from voice_assistant.agent.deep_agent import DeepAgentsAdapter
async def main():
compartment_id = os.environ.get("OCI_COMPARTMENT_ID")
if not compartment_id:
raise RuntimeError("OCI_COMPARTMENT_ID is required.")
adapter = DeepAgentsAdapter(
region="sa-vinhedo-1",
compartment_id=compartment_id,
model_id="openai.gpt-4o-mini",
temperature=0.3,
max_completion_tokens=256,
)
try:
questions = [
"Estou planejando visitar o Japão. Lembre desse destino.",
"Qual país mencionei? Responda somente com o nome.",
]
for turn_id, question in enumerate(questions, 1):
print(f"\n--- Turn {turn_id} ---")
print("User:", question)
chunks = []
async with aclosing(
adapter.stream(
session_id="agentvoice-adapter-smoke",
turn_id=turn_id,
text=question,
)
) as stream:
async for chunk in stream:
chunks.append(chunk)
print(chunk, end="", flush=True)
response = "".join(chunks).strip()
if not response:
raise RuntimeError(
f"Turn {turn_id} returned empty streaming text."
)
print(f"\nChunks: {len(chunks)}")
finally:
await adapter.aclose()
if __name__ == "__main__":
asyncio.run(main())

View File

@@ -0,0 +1,109 @@
"""
Deep Agents contextual memory smoke test.
Uses OCI Generative AI through LangChain.
Does not modify the AgentVoice application.
"""
import asyncio
import os
from deepagents import create_deep_agent
from langchain_oci import ChatOCIGenAI
from langchain_core.messages import HumanMessage
from langgraph.checkpoint.memory import MemorySaver
async def main():
compartment_id = os.environ.get("OCI_COMPARTMENT_ID")
if not compartment_id:
raise RuntimeError("OCI_COMPARTMENT_ID is required.")
model = ChatOCIGenAI(
model_id="openai.gpt-4o-mini",
compartment_id=compartment_id,
service_endpoint=(
"https://inference.generativeai."
"sa-vinhedo-1.oci.oraclecloud.com"
),
auth_type="INSTANCE_PRINCIPAL",
is_stream=True,
model_kwargs={
"temperature": 0.3,
"max_tokens": 256,
},
)
try:
agent = create_deep_agent(
model=model,
tools=[],
system_prompt=(
"Você é o AgentVoice, um assistente por voz. "
"Responda em português brasileiro, de forma "
"natural, breve e contextual. "
"Considere as mensagens anteriores da conversa. "
"Quando o usuário corrigir uma pergunta, "
"interprete a correção usando o histórico."
),
checkpointer=MemorySaver(),
)
config = {
"configurable": {
"thread_id": "agentvoice-context-test",
}
}
questions = [
(
"Estou planejando visitar o Japão. "
"Guarde esse destino no contexto da conversa."
),
(
"Qual foi o país que mencionei anteriormente? "
"Responda somente com o nome do país."
),
]
for number, question in enumerate(questions, 1):
print(f"\n--- Turn {number} ---")
print("User:", question)
result = await agent.ainvoke(
{
"messages": [
HumanMessage(content=question)
]
},
config=config,
)
messages = result.get("messages", [])
response = next(
(
message.content
for message in reversed(messages)
if message.type == "ai"
and isinstance(message.content, str)
and message.content.strip()
),
None,
)
if not response:
raise RuntimeError(
f"Turn {number} returned no AI response."
)
print("Agent:", response)
finally:
await model.aclose()
if __name__ == "__main__":
asyncio.run(main())

View File

@@ -0,0 +1,109 @@
"""
Deep Agents contextual memory smoke test.
Uses OCI Generative AI through LangChain.
Does not modify the AgentVoice application.
"""
import asyncio
import os
from deepagents import create_deep_agent
from langchain_oci import ChatOCIGenAI
from langchain_core.messages import HumanMessage
from langgraph.checkpoint.memory import MemorySaver
async def main():
compartment_id = os.environ.get("OCI_COMPARTMENT_ID")
if not compartment_id:
raise RuntimeError("OCI_COMPARTMENT_ID is required.")
model = ChatOCIGenAI(
model_id="openai.gpt-4o-mini",
compartment_id=compartment_id,
service_endpoint=(
"https://inference.generativeai."
"sa-vinhedo-1.oci.oraclecloud.com"
),
auth_type="INSTANCE_PRINCIPAL",
is_stream=True,
model_kwargs={
"temperature": 0.3,
"max_tokens": 256,
},
)
try:
agent = create_deep_agent(
model=model,
tools=[],
system_prompt=(
"Você é o AgentVoice, um assistente por voz. "
"Responda em português brasileiro, de forma "
"natural, breve e contextual. "
"Considere as mensagens anteriores da conversa. "
"Quando o usuário corrigir uma pergunta, "
"interprete a correção usando o histórico."
),
checkpointer=MemorySaver(),
)
config = {
"configurable": {
"thread_id": "agentvoice-context-test",
}
}
questions = [
(
"Estou planejando visitar o Japão. "
"Guarde esse destino no contexto da conversa."
),
(
"Qual foi o país que mencionei anteriormente? "
"Responda somente com o nome do país."
),
]
for number, question in enumerate(questions, 1):
print(f"\n--- Turn {number} ---")
print("User:", question)
result = await agent.ainvoke(
{
"messages": [
HumanMessage(content=question)
]
},
config=config,
)
messages = result.get("messages", [])
response = next(
(
message.content
for message in reversed(messages)
if message.type == "ai"
and isinstance(message.content, str)
and message.content.strip()
),
None,
)
if not response:
raise RuntimeError(
f"Turn {number} returned no AI response."
)
print("Agent:", response)
finally:
await model.aclose()
if __name__ == "__main__":
asyncio.run(main())

1135
uv.lock generated

File diff suppressed because it is too large Load Diff