primeira versão customizada de VAD- Voice Activity Detection
This commit is contained in:
173
src/voice_assistant/agent/deep_agent.py
Normal file
173
src/voice_assistant/agent/deep_agent.py
Normal file
@@ -0,0 +1,173 @@
|
||||
|
||||
"""
|
||||
Deep Agents implementation of the AgentVoice Agent port.
|
||||
|
||||
Responsibilities:
|
||||
- OCI Generative AI inference through LangChain.
|
||||
- Contextual memory per session.
|
||||
- Text streaming compatible with VoicePipeline.
|
||||
- Asynchronous resource cleanup.
|
||||
|
||||
Does not manage:
|
||||
- WebSocket connections.
|
||||
- STT or TTS.
|
||||
- Semantic turn lifecycle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import aclosing
|
||||
|
||||
from deepagents import create_deep_agent
|
||||
from langchain_core.messages import HumanMessage
|
||||
from langchain_oci import ChatOCIGenAI
|
||||
from langgraph.checkpoint.memory import MemorySaver
|
||||
|
||||
from voice_assistant.agent.adapter import AgentAdapter
|
||||
|
||||
DEFAULT_SYSTEM_PROMPT = """
|
||||
Você é o AgentVoice, um assistente pessoal por voz. Que responde pelo nome de Roberta.
|
||||
|
||||
Responda em português brasileiro, de forma natural,
|
||||
objetiva e adequada para reprodução por voz.
|
||||
|
||||
Mantenha o contexto das mensagens anteriores.
|
||||
Interprete correções e complementos do usuário
|
||||
considerando o histórico da conversa.
|
||||
|
||||
Não invente informações atuais ou resultados
|
||||
de ferramentas que não foram executadas.
|
||||
"""
|
||||
|
||||
|
||||
class DeepAgentsAdapter(AgentAdapter):
|
||||
def __init__(
|
||||
self,
|
||||
region: str,
|
||||
compartment_id: str,
|
||||
model_id: str,
|
||||
temperature: float,
|
||||
max_completion_tokens: int,
|
||||
timeout_seconds: float,
|
||||
system_prompt: str = DEFAULT_SYSTEM_PROMPT,
|
||||
) -> None:
|
||||
if not region:
|
||||
raise ValueError("OCI region is required.")
|
||||
|
||||
if not compartment_id:
|
||||
raise ValueError("OCI compartment_id is required.")
|
||||
|
||||
if not model_id:
|
||||
raise ValueError("OCI model_id is required.")
|
||||
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("timeout_seconds must be positive.")
|
||||
|
||||
self.model = ChatOCIGenAI(
|
||||
model_id=model_id,
|
||||
compartment_id=compartment_id,
|
||||
service_endpoint=(
|
||||
f"https://inference.generativeai."
|
||||
f"{region}.oci.oraclecloud.com"
|
||||
),
|
||||
auth_type="INSTANCE_PRINCIPAL",
|
||||
is_stream=True,
|
||||
model_kwargs={
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_completion_tokens,
|
||||
},
|
||||
)
|
||||
|
||||
self.checkpointer = MemorySaver()
|
||||
|
||||
self.agent = create_deep_agent(
|
||||
model=self.model,
|
||||
tools=[],
|
||||
system_prompt=system_prompt,
|
||||
checkpointer=self.checkpointer,
|
||||
)
|
||||
|
||||
self._closed = False
|
||||
|
||||
def _config(self, session_id: str) -> dict:
|
||||
if not session_id:
|
||||
raise ValueError("session_id is required.")
|
||||
|
||||
return {
|
||||
"configurable": {
|
||||
"thread_id": session_id,
|
||||
}
|
||||
}
|
||||
|
||||
async def process(
|
||||
self,
|
||||
session_id: str,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> str:
|
||||
parts: list[str] = []
|
||||
|
||||
async with aclosing(
|
||||
self.stream(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=text,
|
||||
)
|
||||
) as chunks:
|
||||
async for chunk in chunks:
|
||||
parts.append(chunk)
|
||||
|
||||
response = "".join(parts).strip()
|
||||
|
||||
if not response:
|
||||
raise RuntimeError(
|
||||
"Deep Agent returned an empty response."
|
||||
)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
async def stream(
|
||||
self,
|
||||
session_id: str,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> AsyncIterator[str]:
|
||||
if self._closed:
|
||||
raise RuntimeError("Deep Agent is closed.")
|
||||
|
||||
if not text.strip():
|
||||
raise ValueError("Agent input text cannot be empty.")
|
||||
|
||||
config = self._config(session_id)
|
||||
|
||||
async for message, metadata in self.agent.astream(
|
||||
{
|
||||
"messages": [
|
||||
HumanMessage(content=text)
|
||||
]
|
||||
},
|
||||
config=config,
|
||||
stream_mode="messages",
|
||||
):
|
||||
# Ignore tools and subagent outputs.
|
||||
if metadata.get("langgraph_node") != "model":
|
||||
continue
|
||||
|
||||
# Ignore tool-call fragments.
|
||||
if getattr(message, "tool_call_chunks", None):
|
||||
continue
|
||||
|
||||
content = message.content
|
||||
|
||||
if isinstance(content, str) and content:
|
||||
yield content
|
||||
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
|
||||
self._closed = True
|
||||
await self.model.aclose()
|
||||
@@ -155,6 +155,20 @@ async def send_json(
|
||||
)
|
||||
)
|
||||
|
||||
async def close_optional_provider(provider: object | None) -> None:
|
||||
"""
|
||||
Close providers exposing an optional asynchronous cleanup hook.
|
||||
|
||||
Providers without aclose() require no action.
|
||||
"""
|
||||
if provider is None:
|
||||
return
|
||||
|
||||
close_method = getattr(provider, "aclose", None)
|
||||
|
||||
if callable(close_method):
|
||||
await close_method()
|
||||
|
||||
|
||||
async def forward_stt_events(
|
||||
websocket: WebSocket,
|
||||
@@ -331,13 +345,29 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
logger.exception(
|
||||
"Turn finalization failed. session_id=%s turn_id=%s",
|
||||
session.session_id,
|
||||
turn_id,
|
||||
)
|
||||
|
||||
if turn_id not in interrupting_turns:
|
||||
try:
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"TURN_EXECUTION_FAILED",
|
||||
str(exc),
|
||||
session_id=session.session_id,
|
||||
turn_id=turn_id,
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to report turn execution failure."
|
||||
)
|
||||
|
||||
async def cancel_finalization_task() -> None:
|
||||
"""
|
||||
Cancel and await the Gateway-owned finalization task.
|
||||
@@ -792,7 +822,6 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
|
||||
continue
|
||||
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# audio.stop
|
||||
# ---------------------------------------------------------
|
||||
@@ -805,37 +834,41 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
"STT session has not been started."
|
||||
)
|
||||
|
||||
if pending_audio_chunk is not None:
|
||||
raise ValueError(
|
||||
"audio.stop received while an audio chunk is pending."
|
||||
)
|
||||
|
||||
if agent is None:
|
||||
raise RuntimeError(
|
||||
"Agent adapter is not initialized."
|
||||
)
|
||||
|
||||
if tts is None:
|
||||
raise RuntimeError(
|
||||
"TTS adapter is not initialized."
|
||||
)
|
||||
|
||||
if pipeline is None:
|
||||
raise RuntimeError(
|
||||
"Voice pipeline is not initialized."
|
||||
)
|
||||
|
||||
# Finish the STT session normally.
|
||||
await stt.stop_session()
|
||||
await pipeline.wait_turn_task()
|
||||
stt_events_task = None
|
||||
|
||||
try:
|
||||
pipeline.complete_turn(request.turn_id)
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
if agent is None or tts is None:
|
||||
raise RuntimeError(
|
||||
"Turn pipeline did not complete successfully."
|
||||
) from exc
|
||||
"Agent or TTS adapter is not initialized."
|
||||
)
|
||||
|
||||
if not pipeline.is_active_turn(request.turn_id):
|
||||
raise ValueError(
|
||||
f"Unexpected turn_id={request.turn_id}."
|
||||
)
|
||||
|
||||
if pending_audio_chunk is not None:
|
||||
raise ValueError(
|
||||
"audio.stop received while an audio chunk is pending."
|
||||
)
|
||||
|
||||
if (
|
||||
finalization_task is not None
|
||||
and not finalization_task.done()
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Turn finalization is already running."
|
||||
)
|
||||
|
||||
await stt.stop_session()
|
||||
|
||||
# Do not await OCI/TTS here.
|
||||
# Keep the WebSocket receive loop available.
|
||||
finalization_task = asyncio.create_task(
|
||||
finalize_turn(request.turn_id)
|
||||
)
|
||||
|
||||
except (ValidationError, RuntimeError, ValueError) as exc:
|
||||
await send_json(
|
||||
@@ -847,20 +880,10 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
turn_id=payload.get("turn_id"),
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
stt = None
|
||||
pending_audio_chunk = None
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
TurnEnded(
|
||||
session_id=session.session_id,
|
||||
turn_id=request.turn_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
|
||||
# ---------------------------------------------------------
|
||||
# interrupt
|
||||
# ---------------------------------------------------------
|
||||
@@ -895,8 +918,11 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
# Only cancel resources belonging to the requested active turn.
|
||||
if pipeline.is_active_turn(request.turn_id):
|
||||
try:
|
||||
# Cancel and await STT/Agent/TTS execution before
|
||||
# confirming the interruption to the client.
|
||||
async with turn_transition_lock:
|
||||
interrupting_turns.add(request.turn_id)
|
||||
|
||||
await cancel_finalization_task()
|
||||
|
||||
await pipeline.cancel_turn_task()
|
||||
|
||||
# Abort transcription without requesting a final result.
|
||||
@@ -923,8 +949,19 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
continue
|
||||
|
||||
else:
|
||||
# Preserve the existing tolerant behavior for unknown turns.
|
||||
pipeline.interrupt_if_active(request.turn_id)
|
||||
await send_json(
|
||||
websocket,
|
||||
error_event(
|
||||
"INVALID_TURN",
|
||||
(
|
||||
f"Turn {request.turn_id} "
|
||||
"is no longer active."
|
||||
),
|
||||
session_id=session.session_id,
|
||||
turn_id=request.turn_id,
|
||||
),
|
||||
)
|
||||
continue
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
@@ -946,6 +983,12 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
raise RuntimeError(
|
||||
"Voice pipeline is not initialized."
|
||||
)
|
||||
# Prevent normal completion during session shutdown.
|
||||
async with turn_transition_lock:
|
||||
if session.active_turn_id is not None:
|
||||
interrupting_turns.add(session.active_turn_id)
|
||||
|
||||
await cancel_finalization_task()
|
||||
|
||||
# Cancel and await any running STT/Agent/TTS task.
|
||||
await pipeline.cancel_turn_task()
|
||||
@@ -1079,3 +1122,15 @@ async def audio_socket(websocket: WebSocket) -> None:
|
||||
session.session_id if session else None,
|
||||
)
|
||||
|
||||
# Release provider-owned HTTP connections.
|
||||
if agent is not None:
|
||||
try:
|
||||
await close_optional_provider(agent)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to close Agent provider. session_id=%s",
|
||||
session.session_id if session else None,
|
||||
)
|
||||
finally:
|
||||
agent = None
|
||||
|
||||
|
||||
@@ -82,8 +82,12 @@ export class PCMPlayer {
|
||||
}
|
||||
|
||||
async enqueue(buffer) {
|
||||
const generation = this.generation;
|
||||
|
||||
if (this.finished) {
|
||||
throw new Error("Playback has already been finalized.");
|
||||
// Playback was already stopped or finalized.
|
||||
// Late audio chunks must not terminate the conversation.
|
||||
return;
|
||||
}
|
||||
|
||||
const samples = this.decodePCM(buffer);
|
||||
@@ -94,8 +98,12 @@ export class PCMPlayer {
|
||||
|
||||
await this.context.resume();
|
||||
|
||||
if (generation !== this.generation) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.finished) {
|
||||
throw new Error("Playback was finalized during enqueue.");
|
||||
return;
|
||||
}
|
||||
|
||||
const audioBuffer = this.context.createBuffer(
|
||||
@@ -116,9 +124,6 @@ export class PCMPlayer {
|
||||
);
|
||||
|
||||
this.nextStartTime = startAt + audioBuffer.duration;
|
||||
|
||||
const generation = this.generation;
|
||||
|
||||
this.sources.add(source);
|
||||
|
||||
if (!this.playing) {
|
||||
|
||||
@@ -47,6 +47,22 @@ export class VoiceSessionController {
|
||||
this.playbackPending = Promise.resolve();
|
||||
this.gatewayTurnEnded = false;
|
||||
this.playbackGeneration = 0;
|
||||
this.bargeInPending = false;
|
||||
this.bargeInBuffer = [];
|
||||
this.maxBargeInChunks = 100;
|
||||
this.bargeInSpeechEnded = false;
|
||||
this.interruptedTurnId = null;
|
||||
this.bargeInCaptureStarted = false;
|
||||
|
||||
this.bargeInVad = new VoiceActivityDetector({
|
||||
minSpeechMs: 350,
|
||||
silenceMs: 800,
|
||||
threshold: 0.025,
|
||||
noiseMultiplier: 4,
|
||||
|
||||
onSpeechStart: () => this.requestBargeIn(),
|
||||
onSpeechEnd: () => this.finishBargeInSpeech(),
|
||||
});
|
||||
}
|
||||
|
||||
getPlaybackAnalyser() {
|
||||
@@ -54,6 +70,16 @@ export class VoiceSessionController {
|
||||
}
|
||||
|
||||
setState(state) {
|
||||
console.debug(
|
||||
`[AgentVoice][STATE] ${this.state} -> ${state}`,
|
||||
{
|
||||
turnId: this.turnId,
|
||||
playerFinished: this.player?.finished,
|
||||
bargeInPending: this.bargeInPending,
|
||||
bargeInRecording: this.bargeInRecording,
|
||||
}
|
||||
);
|
||||
|
||||
this.state = state;
|
||||
this.onStateChange(state);
|
||||
}
|
||||
@@ -78,11 +104,16 @@ export class VoiceSessionController {
|
||||
audioContext,
|
||||
sampleRate: 48000,
|
||||
|
||||
onPlaybackStart: () => {
|
||||
if (this.state !== "closed") {
|
||||
this.setState("speaking");
|
||||
}
|
||||
},
|
||||
onPlaybackStart: () => {
|
||||
if (
|
||||
this.state === "processing" ||
|
||||
this.state === "speaking"
|
||||
) {
|
||||
this.setState("speaking");
|
||||
this.bargeInVad.start();
|
||||
this.monitorVoice();
|
||||
}
|
||||
},
|
||||
});
|
||||
this.rmsSamples = new Float32Array(analyser.fftSize);
|
||||
|
||||
@@ -162,67 +193,102 @@ export class VoiceSessionController {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
startListening() {
|
||||
this.playbackGeneration++;
|
||||
|
||||
this.gatewayTurnEnded = false;
|
||||
this.player?.reset();
|
||||
this.playbackPending = Promise.resolve();
|
||||
this.playbackGeneration++;
|
||||
|
||||
this.turnId = null;
|
||||
this.sequence = 0;
|
||||
this.prebuffer = [];
|
||||
this.pending = [];
|
||||
this.stopRequested = false;
|
||||
this.gatewayTurnEnded = false;
|
||||
this.player?.reset();
|
||||
this.playbackPending = Promise.resolve();
|
||||
|
||||
this.worklet.port.postMessage({ type: "start" });
|
||||
this.vad.start();
|
||||
this.setState("listening");
|
||||
this.bargeInVad.stop();
|
||||
this.bargeInPending = false;
|
||||
this.bargeInSpeechEnded = false;
|
||||
this.interruptedTurnId = null;
|
||||
this.bargeInBuffer = [];
|
||||
|
||||
this.monitorVoice();
|
||||
this.turnId = null;
|
||||
this.sequence = 0;
|
||||
this.prebuffer = [];
|
||||
this.pending = [];
|
||||
this.stopRequested = false;
|
||||
|
||||
this.worklet.port.postMessage({ type: "start" });
|
||||
|
||||
this.vad.start();
|
||||
this.setState("listening");
|
||||
this.monitorVoice();
|
||||
}
|
||||
|
||||
monitorVoice() {
|
||||
cancelAnimationFrame(this.frameId);
|
||||
|
||||
const tick = (now) => {
|
||||
if (this.state !== "listening" &&
|
||||
this.state !== "starting" &&
|
||||
this.state !== "recording") {
|
||||
return;
|
||||
}
|
||||
monitorVoice() {
|
||||
cancelAnimationFrame(this.frameId);
|
||||
|
||||
this.analyser.getFloatTimeDomainData(this.rmsSamples);
|
||||
const tick = (now) => {
|
||||
const normalListening = [
|
||||
"listening",
|
||||
"starting",
|
||||
"recording"
|
||||
].includes(this.state);
|
||||
|
||||
let power = 0;
|
||||
for (const sample of this.rmsSamples) {
|
||||
power += sample * sample;
|
||||
}
|
||||
const bargeInListening = [
|
||||
"processing",
|
||||
"speaking",
|
||||
"waiting-playback",
|
||||
"interrupting"
|
||||
].includes(this.state) || this.bargeInRecording;
|
||||
|
||||
const rms = Math.sqrt(power / this.rmsSamples.length);
|
||||
if (!normalListening && !bargeInListening) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!this.stopRequested) {
|
||||
this.vad.process(rms, now);
|
||||
}
|
||||
this.analyser.getFloatTimeDomainData(this.rmsSamples);
|
||||
|
||||
let power = 0;
|
||||
|
||||
for (const sample of this.rmsSamples) {
|
||||
power += sample * sample;
|
||||
}
|
||||
|
||||
const rms = Math.sqrt(
|
||||
power / this.rmsSamples.length
|
||||
);
|
||||
|
||||
// Normal speech detection.
|
||||
if (
|
||||
normalListening &&
|
||||
!this.bargeInRecording &&
|
||||
!this.stopRequested
|
||||
) {
|
||||
this.vad.process(rms, now);
|
||||
}
|
||||
|
||||
// Continue tracking speech initiated during TTS.
|
||||
if (
|
||||
bargeInListening &&
|
||||
this.bargeInVad.active
|
||||
) {
|
||||
this.bargeInVad.process(rms, now);
|
||||
}
|
||||
|
||||
this.frameId = requestAnimationFrame(tick);
|
||||
};
|
||||
|
||||
this.frameId = requestAnimationFrame(tick);
|
||||
};
|
||||
|
||||
this.frameId = requestAnimationFrame(tick);
|
||||
}
|
||||
|
||||
beginTurn() {
|
||||
if (this.state !== "listening") return;
|
||||
|
||||
this.pending = [...this.prebuffer];
|
||||
this.setState("starting");
|
||||
|
||||
try {
|
||||
this.send({ type: "audio.start" });
|
||||
} catch (error) {
|
||||
this.fail(error);
|
||||
}
|
||||
}
|
||||
|
||||
beginTurn() {
|
||||
if (this.state !== "listening") return;
|
||||
|
||||
this.pending = [...this.prebuffer];
|
||||
this.setState("starting");
|
||||
|
||||
try {
|
||||
this.send({ type: "audio.start" });
|
||||
} catch (error) {
|
||||
this.fail(error);
|
||||
}
|
||||
}
|
||||
|
||||
handleWorkletMessage(data) {
|
||||
if (data?.type === "pcm") {
|
||||
@@ -242,6 +308,21 @@ export class VoiceSessionController {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
this.state === "processing" ||
|
||||
this.state === "speaking" ||
|
||||
this.state === "waiting-playback" ||
|
||||
this.state === "interrupting"
|
||||
) {
|
||||
this.bargeInBuffer.push(buffer);
|
||||
|
||||
if (this.bargeInBuffer.length > this.maxBargeInChunks) {
|
||||
this.bargeInBuffer.shift();
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.state === "starting") {
|
||||
this.pending.push(buffer);
|
||||
|
||||
@@ -302,31 +383,80 @@ export class VoiceSessionController {
|
||||
});
|
||||
|
||||
this.setState("processing");
|
||||
cancelAnimationFrame(this.frameId);
|
||||
|
||||
// Resume local PCM capture for a possible interruption.
|
||||
// Audio will remain buffered until a new turn is accepted.
|
||||
this.worklet.port.postMessage({ type: "start" });
|
||||
|
||||
// The microphone remains available for barge-in.
|
||||
// Detection starts when TTS playback begins.
|
||||
this.monitorVoice();
|
||||
}
|
||||
|
||||
handleAudioChunk(buffer) {
|
||||
|
||||
handleAudioChunk(buffer) {
|
||||
if (
|
||||
this.state === "closed" ||
|
||||
this.turnId === null ||
|
||||
this.gatewayTurnEnded
|
||||
this.state === "closed" ||
|
||||
this.state === "interrupting" ||
|
||||
this.bargeInPending ||
|
||||
this.turnId === null ||
|
||||
this.gatewayTurnEnded
|
||||
) {
|
||||
return;
|
||||
return;
|
||||
}
|
||||
|
||||
console.debug("[AgentVoice][PCM recebido]", {
|
||||
bytes: buffer.byteLength,
|
||||
turnId: this.turnId,
|
||||
state: this.state,
|
||||
generation: this.playbackGeneration,
|
||||
finished: this.player?.finished,
|
||||
gatewayTurnEnded: this.gatewayTurnEnded,
|
||||
});
|
||||
|
||||
const generation = this.playbackGeneration;
|
||||
|
||||
this.playbackPending = this.playbackPending
|
||||
.then(async () => {
|
||||
if (generation !== this.playbackGeneration) return;
|
||||
await this.player.enqueue(buffer);
|
||||
})
|
||||
.catch((error) => {
|
||||
if (generation === this.playbackGeneration) {
|
||||
this.fail(error);
|
||||
.then(async () => {
|
||||
if (generation !== this.playbackGeneration) {
|
||||
return;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
this.state === "interrupting" ||
|
||||
this.bargeInPending
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.debug("[AgentVoice][PCM enqueue]", {
|
||||
turnId: this.turnId,
|
||||
state: this.state,
|
||||
generation,
|
||||
currentGeneration: this.playbackGeneration,
|
||||
finished: this.player?.finished,
|
||||
});
|
||||
|
||||
|
||||
await this.player.enqueue(buffer);
|
||||
})
|
||||
.catch((error) => {
|
||||
// An obsolete playback must never terminate the session.
|
||||
if (generation !== this.playbackGeneration) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
this.state === "interrupting" ||
|
||||
this.bargeInPending
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.fail(error);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async finishPlaybackAndResume() {
|
||||
const generation = this.playbackGeneration;
|
||||
@@ -353,6 +483,66 @@ export class VoiceSessionController {
|
||||
}
|
||||
|
||||
|
||||
requestBargeIn() {
|
||||
if (this.bargeInPending) return;
|
||||
|
||||
if (
|
||||
this.state !== "speaking" &&
|
||||
this.state !== "waiting-playback"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.turnId === null) return;
|
||||
|
||||
const interruptedTurnId = this.turnId;
|
||||
|
||||
this.bargeInPending = true;
|
||||
this.bargeInSpeechEnded = false;
|
||||
this.bargeInCaptureStarted = true;
|
||||
this.interruptedTurnId = interruptedTurnId;
|
||||
|
||||
// Stop local playback immediately.
|
||||
this.playbackGeneration++;
|
||||
this.player?.stop();
|
||||
this.playbackPending = Promise.resolve();
|
||||
|
||||
// Prevent normal playback completion from resuming listening.
|
||||
this.gatewayTurnEnded = false;
|
||||
|
||||
// The normal VAD must not start a competing turn.
|
||||
this.vad.stop();
|
||||
|
||||
this.setState("interrupting");
|
||||
|
||||
try {
|
||||
this.send({
|
||||
type: "interrupt",
|
||||
turn_id: interruptedTurnId,
|
||||
});
|
||||
} catch (error) {
|
||||
this.fail(error);
|
||||
}
|
||||
}
|
||||
|
||||
finishBargeInSpeech() {
|
||||
if (!this.bargeInPending && !this.bargeInRecording) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.bargeInSpeechEnded = true;
|
||||
|
||||
if (
|
||||
this.bargeInRecording &&
|
||||
this.state === "recording"
|
||||
) {
|
||||
this.bargeInRecording = false;
|
||||
this.bargeInVad.stop();
|
||||
|
||||
this.stopRequested = true;
|
||||
this.flushTurn();
|
||||
}
|
||||
}
|
||||
|
||||
handleGatewayEvent(event) {
|
||||
switch (event.type) {
|
||||
@@ -360,9 +550,39 @@ export class VoiceSessionController {
|
||||
this.startListening();
|
||||
break;
|
||||
|
||||
// Debug logging for all events received from the Gateway.
|
||||
console.debug("[AgentVoice][Gateway]", {
|
||||
type: event.type,
|
||||
turnId: event.turn_id,
|
||||
currentTurnId: this.turnId,
|
||||
state: this.state,
|
||||
playbackGeneration: this.playbackGeneration,
|
||||
playerFinished: this.player?.finished,
|
||||
playerPlaying: this.player?.playing,
|
||||
bargeInPending: this.bargeInPending,
|
||||
bargeInRecording: this.bargeInRecording,
|
||||
text: event.type === "stt.final"
|
||||
? event.text
|
||||
: event.type === "agent.response"
|
||||
? event.text
|
||||
: undefined,
|
||||
error: event.type === "error"
|
||||
? { code: event.code, message: event.message }
|
||||
: undefined,
|
||||
});
|
||||
// Debug logging for all events received from the Gateway.
|
||||
|
||||
|
||||
case "turn.started":
|
||||
if (this.state !== "starting") return;
|
||||
|
||||
// Prepare playback for the newly accepted turn.
|
||||
// The previous TTS may have been stopped by barge-in.
|
||||
this.playbackGeneration++;
|
||||
this.player?.reset();
|
||||
this.playbackPending = Promise.resolve();
|
||||
this.gatewayTurnEnded = false;
|
||||
|
||||
this.turnId = event.turn_id;
|
||||
this.sequence = 0;
|
||||
|
||||
@@ -374,6 +594,25 @@ export class VoiceSessionController {
|
||||
|
||||
this.pending = [];
|
||||
|
||||
if (this.bargeInPending) {
|
||||
this.bargeInPending = false;
|
||||
this.bargeInCaptureStarted = false;
|
||||
|
||||
if (this.bargeInSpeechEnded) {
|
||||
// User finished speaking before turn.started.
|
||||
this.bargeInRecording = false;
|
||||
this.bargeInVad.stop();
|
||||
this.stopRequested = true;
|
||||
} else {
|
||||
// User is still speaking.
|
||||
// Preserve the active barge-in VAD state.
|
||||
this.bargeInRecording = true;
|
||||
this.stopRequested = false;
|
||||
}
|
||||
|
||||
this.bargeInSpeechEnded = false;
|
||||
}
|
||||
|
||||
if (this.stopRequested) {
|
||||
this.flushTurn();
|
||||
}
|
||||
@@ -399,19 +638,71 @@ export class VoiceSessionController {
|
||||
break;
|
||||
|
||||
case "turn.ended":
|
||||
if (event.turn_id === this.turnId) {
|
||||
this.gatewayTurnEnded = true;
|
||||
this.setState("waiting-playback");
|
||||
if (
|
||||
event.turn_id === this.turnId &&
|
||||
!this.bargeInPending
|
||||
) {
|
||||
this.gatewayTurnEnded = true;
|
||||
this.setState("waiting-playback");
|
||||
|
||||
void this.finishPlaybackAndResume();
|
||||
void this.finishPlaybackAndResume();
|
||||
}
|
||||
break;
|
||||
|
||||
case "interrupted":
|
||||
this.close();
|
||||
if (
|
||||
!this.bargeInPending ||
|
||||
event.turn_id !== this.interruptedTurnId
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.interruptedTurnId = null;
|
||||
this.turnId = null;
|
||||
this.sequence = 0;
|
||||
this.pending = [...this.bargeInBuffer];
|
||||
this.bargeInBuffer = [];
|
||||
|
||||
this.stopRequested = this.bargeInSpeechEnded;
|
||||
|
||||
this.setState("starting");
|
||||
|
||||
try {
|
||||
this.send({ type: "audio.start" });
|
||||
} catch (error) {
|
||||
this.fail(error);
|
||||
}
|
||||
break;
|
||||
|
||||
case "error":
|
||||
if (
|
||||
event.code === "INVALID_TURN" &&
|
||||
this.bargeInPending &&
|
||||
event.turn_id === this.interruptedTurnId
|
||||
) {
|
||||
// The old turn completed before interrupt was processed.
|
||||
// Its semantic state is already released by the Gateway.
|
||||
this.interruptedTurnId = null;
|
||||
this.turnId = null;
|
||||
this.sequence = 0;
|
||||
|
||||
this.pending = [...this.bargeInBuffer];
|
||||
this.bargeInBuffer = [];
|
||||
|
||||
this.stopRequested = this.bargeInSpeechEnded;
|
||||
this.gatewayTurnEnded = false;
|
||||
|
||||
this.setState("starting");
|
||||
|
||||
try {
|
||||
this.send({ type: "audio.start" });
|
||||
} catch (error) {
|
||||
this.fail(error);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
this.fail(
|
||||
new Error(`${event.code}: ${event.message}`)
|
||||
);
|
||||
@@ -435,6 +726,14 @@ export class VoiceSessionController {
|
||||
this.player?.stop();
|
||||
|
||||
this.vad.stop();
|
||||
this.bargeInVad.stop();
|
||||
|
||||
this.bargeInPending = false;
|
||||
this.bargeInSpeechEnded = false;
|
||||
this.bargeInCaptureStarted = false;
|
||||
this.interruptedTurnId = null;
|
||||
this.bargeInBuffer = [];
|
||||
|
||||
cancelAnimationFrame(this.frameId);
|
||||
|
||||
if (this.worklet) {
|
||||
|
||||
Reference in New Issue
Block a user