feat: integrate OCI agent streaming and web voice client
This commit is contained in:
@@ -10,7 +10,6 @@ from __future__ import annotations
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import aclosing
|
||||
@@ -19,6 +18,7 @@ import oci
|
||||
|
||||
from voice_assistant.agent.adapter import AgentAdapter
|
||||
from voice_assistant.agent.oci_stream import stream_sse_events
|
||||
from voice_assistant.agent.oci_connection import open_oci_stream
|
||||
|
||||
|
||||
def extract_text_delta(raw_data: str) -> str | None:
|
||||
@@ -205,9 +205,8 @@ class OCIAgentAdapter(AgentAdapter):
|
||||
self.model_id,
|
||||
)
|
||||
|
||||
response = await asyncio.to_thread(
|
||||
self.client.chat,
|
||||
details,
|
||||
response = await open_oci_stream(
|
||||
lambda: self.client.chat(details)
|
||||
)
|
||||
|
||||
if response.status != 200:
|
||||
@@ -256,4 +255,3 @@ class OCIAgentAdapter(AgentAdapter):
|
||||
)
|
||||
|
||||
return response_text
|
||||
|
||||
|
||||
107
src/voice_assistant/agent/oci_connection.py
Normal file
107
src/voice_assistant/agent/oci_connection.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""
|
||||
Lifecycle management for blocking OCI HTTP connection setup.
|
||||
|
||||
Ensures that responses arriving after coroutine cancellation
|
||||
are closed instead of being abandoned.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _close_response(response: Any) -> None:
|
||||
"""Close the SSE response returned by the OCI SDK."""
|
||||
stream = getattr(response, "data", None)
|
||||
|
||||
if stream is not None:
|
||||
close = getattr(stream, "close", None)
|
||||
|
||||
if callable(close):
|
||||
close()
|
||||
|
||||
|
||||
async def open_oci_stream(
|
||||
open_connection: Callable[[], Any],
|
||||
) -> Any:
|
||||
"""
|
||||
Open a blocking OCI connection with explicit response ownership.
|
||||
|
||||
The worker retains ownership until the async caller claims
|
||||
the response. Cancellation closes any unclaimed response.
|
||||
|
||||
A blocking SDK call may continue after cancellation.
|
||||
"""
|
||||
ownership_lock = threading.Lock()
|
||||
abandoned = False
|
||||
claimed = False
|
||||
response = None
|
||||
|
||||
def worker() -> Any:
|
||||
nonlocal response
|
||||
|
||||
opened = open_connection()
|
||||
|
||||
with ownership_lock:
|
||||
if abandoned:
|
||||
should_close = True
|
||||
else:
|
||||
response = opened
|
||||
should_close = False
|
||||
|
||||
if should_close:
|
||||
try:
|
||||
_close_response(opened)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to close abandoned OCI response."
|
||||
)
|
||||
|
||||
return opened
|
||||
|
||||
task = asyncio.create_task(
|
||||
asyncio.to_thread(worker)
|
||||
)
|
||||
|
||||
try:
|
||||
opened = await asyncio.shield(task)
|
||||
|
||||
with ownership_lock:
|
||||
if abandoned:
|
||||
raise RuntimeError(
|
||||
"OCI connection was abandoned."
|
||||
)
|
||||
|
||||
claimed = True
|
||||
|
||||
return opened
|
||||
|
||||
except asyncio.CancelledError:
|
||||
with ownership_lock:
|
||||
abandoned = True
|
||||
|
||||
if not claimed:
|
||||
unclaimed = response
|
||||
response = None
|
||||
else:
|
||||
unclaimed = None
|
||||
|
||||
if unclaimed is not None:
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
_close_response,
|
||||
unclaimed,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to close unclaimed OCI response."
|
||||
)
|
||||
|
||||
raise
|
||||
|
||||
85
src/voice_assistant/agent/text_chunker.py
Normal file
85
src/voice_assistant/agent/text_chunker.py
Normal file
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
Incremental text segmentation for streaming voice synthesis.
|
||||
|
||||
Converts LLM text deltas into speech-ready segments while
|
||||
preserving word boundaries and punctuation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class TextChunker:
|
||||
"""
|
||||
Accumulate text deltas and emit segments suitable for TTS.
|
||||
|
||||
Segments are emitted on sentence-ending punctuation or
|
||||
at a configurable maximum length.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
min_chars: int = 20,
|
||||
max_chars: int = 160,
|
||||
) -> None:
|
||||
if min_chars < 1:
|
||||
raise ValueError("min_chars must be positive.")
|
||||
|
||||
if max_chars < min_chars:
|
||||
raise ValueError(
|
||||
"max_chars must be greater than or equal to min_chars."
|
||||
)
|
||||
|
||||
self.min_chars = min_chars
|
||||
self.max_chars = max_chars
|
||||
self._buffer = ""
|
||||
|
||||
def feed(self, delta: str) -> list[str]:
|
||||
"""Accept a text delta and return completed segments."""
|
||||
self._buffer += delta
|
||||
segments: list[str] = []
|
||||
|
||||
while len(self._buffer) >= self.min_chars:
|
||||
boundary = self._find_boundary()
|
||||
|
||||
if boundary is None:
|
||||
break
|
||||
|
||||
segment = self._buffer[:boundary].strip()
|
||||
self._buffer = self._buffer[boundary:]
|
||||
|
||||
if segment:
|
||||
segments.append(segment)
|
||||
|
||||
return segments
|
||||
|
||||
def _find_boundary(self) -> int | None:
|
||||
"""Find a sentence boundary or safe length boundary."""
|
||||
for index, char in enumerate(self._buffer):
|
||||
if char in ".!?":
|
||||
boundary = index + 1
|
||||
|
||||
if boundary >= self.min_chars:
|
||||
return boundary
|
||||
|
||||
if len(self._buffer) < self.max_chars:
|
||||
return None
|
||||
|
||||
# Prefer breaking at a word boundary.
|
||||
boundary = self._buffer.rfind(
|
||||
" ",
|
||||
0,
|
||||
self.max_chars + 1,
|
||||
)
|
||||
|
||||
if boundary >= self.min_chars:
|
||||
return boundary + 1
|
||||
|
||||
return self.max_chars
|
||||
|
||||
def flush(self) -> str | None:
|
||||
"""Return any remaining text after generation completes."""
|
||||
remaining = self._buffer.strip()
|
||||
self._buffer = ""
|
||||
|
||||
return remaining or None
|
||||
|
||||
@@ -34,8 +34,13 @@ Architecture:
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from contextlib import aclosing
|
||||
|
||||
import asyncio
|
||||
|
||||
@@ -83,8 +88,16 @@ from voice_assistant.audio_gateway.app.protocol import (
|
||||
STTPartial,
|
||||
TurnEnded,
|
||||
TurnStarted,
|
||||
AgentDelta,
|
||||
)
|
||||
|
||||
from voice_assistant.voice_pipeline import (
|
||||
PipelineTextDelta,
|
||||
PipelineAgentResponse,
|
||||
PipelineAudioChunk,
|
||||
)
|
||||
|
||||
|
||||
from voice_assistant.audio_gateway.app.session import SessionState
|
||||
|
||||
# logger configuration
|
||||
@@ -100,6 +113,18 @@ provider_catalog = load_provider_catalog()
|
||||
|
||||
MAX_AUDIO_CHUNK_BYTES = 64 * 1024 # 64 KB
|
||||
|
||||
WEB_DIR = Path(__file__).resolve().parents[1] / "web"
|
||||
|
||||
app.mount(
|
||||
"/web",
|
||||
StaticFiles(directory=WEB_DIR),
|
||||
name="web",
|
||||
)
|
||||
|
||||
|
||||
@app.get("/ui", include_in_schema=False)
|
||||
async def web_client() -> FileResponse:
|
||||
return FileResponse(WEB_DIR / "index.html")
|
||||
|
||||
|
||||
def create_turn_manager() -> TurnManager:
|
||||
@@ -176,27 +201,63 @@ async def forward_stt_events(
|
||||
),
|
||||
)
|
||||
|
||||
result = await pipeline.process_transcript(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=event.text,
|
||||
)
|
||||
if callable(getattr(pipeline.agent, "stream", None)):
|
||||
async with aclosing(
|
||||
pipeline.stream_turn_response(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=event.text,
|
||||
)
|
||||
) as response_events:
|
||||
async for response_event in response_events:
|
||||
if not pipeline.is_active_turn(turn_id):
|
||||
break
|
||||
|
||||
await send_json(
|
||||
websocket,
|
||||
AgentResponse(
|
||||
if isinstance(response_event, PipelineTextDelta):
|
||||
await send_json(
|
||||
websocket,
|
||||
AgentDelta(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=response_event.text,
|
||||
),
|
||||
)
|
||||
|
||||
elif isinstance(response_event, PipelineAgentResponse):
|
||||
await send_json(
|
||||
websocket,
|
||||
AgentResponse(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=response_event.text,
|
||||
),
|
||||
)
|
||||
|
||||
elif isinstance(response_event, PipelineAudioChunk):
|
||||
await websocket.send_bytes(response_event.audio)
|
||||
|
||||
else:
|
||||
result = await pipeline.process_transcript(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=result.response_text,
|
||||
),
|
||||
)
|
||||
text=event.text,
|
||||
)
|
||||
|
||||
async for audio_chunk in pipeline.synthesize_response(
|
||||
result.response_text,
|
||||
):
|
||||
await websocket.send_bytes(audio_chunk)
|
||||
await send_json(
|
||||
websocket,
|
||||
AgentResponse(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=result.response_text,
|
||||
),
|
||||
)
|
||||
|
||||
pipeline.on_tts_complete(turn_id)
|
||||
async for audio_chunk in pipeline.synthesize_response(
|
||||
result.response_text,
|
||||
):
|
||||
await websocket.send_bytes(audio_chunk)
|
||||
|
||||
pipeline.on_tts_complete(turn_id)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -109,6 +109,20 @@ class STTFinal(BaseModel):
|
||||
text: str
|
||||
|
||||
|
||||
class AgentDelta(BaseModel):
|
||||
"""
|
||||
Incremental text generated by the configured Agent.
|
||||
|
||||
Each event belongs to one session and turn.
|
||||
Text fragments must be concatenated in arrival order.
|
||||
"""
|
||||
|
||||
type: Literal["agent.delta"] = "agent.delta"
|
||||
session_id: str
|
||||
turn_id: int
|
||||
text: str
|
||||
|
||||
|
||||
class AgentResponse(BaseModel):
|
||||
"""Text response produced by the configured Agent."""
|
||||
|
||||
|
||||
387
src/voice_assistant/audio_gateway/web/app.js
Normal file
387
src/voice_assistant/audio_gateway/web/app.js
Normal file
@@ -0,0 +1,387 @@
|
||||
|
||||
const $ = (id) => document.getElementById(id);
|
||||
|
||||
const status = $("status");
|
||||
const talkButton = $("talkButton");
|
||||
const talkLabel = $("talkLabel");
|
||||
const connectButton = $("connectButton");
|
||||
const interruptButton = $("interruptButton");
|
||||
const messages = $("messages");
|
||||
|
||||
let ws = null;
|
||||
let sessionId = null;
|
||||
let turnId = null;
|
||||
let sequence = 0;
|
||||
let phase = "disconnected";
|
||||
|
||||
let microphone = null;
|
||||
let inputContext = null;
|
||||
let inputSource = null;
|
||||
let inputNode = null;
|
||||
let silentGain = null;
|
||||
|
||||
let playbackContext = null;
|
||||
let playbackTime = 0;
|
||||
const scheduledAudio = new Set();
|
||||
|
||||
let userMessage = null;
|
||||
let agentMessage = null;
|
||||
let pendingFinal = null;
|
||||
|
||||
function setPhase(next) {
|
||||
phase = next;
|
||||
|
||||
status.textContent = {
|
||||
disconnected: "Desconectado",
|
||||
connecting: "Conectando",
|
||||
ready: "Conectado",
|
||||
starting: "Iniciando turno",
|
||||
recording: "Gravando",
|
||||
processing: "Processando",
|
||||
speaking: "Respondendo"
|
||||
}[next] || next;
|
||||
|
||||
status.classList.toggle(
|
||||
"connected",
|
||||
next !== "disconnected" && next !== "connecting"
|
||||
);
|
||||
|
||||
connectButton.disabled = next !== "disconnected";
|
||||
talkButton.disabled = next !== "ready" && next !== "recording";
|
||||
interruptButton.disabled = turnId === null;
|
||||
|
||||
talkButton.classList.toggle("recording", next === "recording");
|
||||
talkLabel.textContent = next === "recording"
|
||||
? "Clique para terminar de falar"
|
||||
: next === "ready"
|
||||
? "Clique para falar"
|
||||
: status.textContent;
|
||||
}
|
||||
|
||||
function addMessage(role, text = "") {
|
||||
$("emptyState")?.remove();
|
||||
|
||||
const element = document.createElement("div");
|
||||
element.className = `message ${role}`;
|
||||
|
||||
const label = document.createElement("small");
|
||||
label.textContent = role === "user" ? "Você" : "AgentVoice";
|
||||
|
||||
const content = document.createElement("span");
|
||||
content.textContent = text;
|
||||
|
||||
element.append(label, content);
|
||||
messages.appendChild(element);
|
||||
messages.scrollTop = messages.scrollHeight;
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
function sendJson(payload) {
|
||||
if (ws?.readyState !== WebSocket.OPEN) {
|
||||
throw new Error("WebSocket não está conectado.");
|
||||
}
|
||||
ws.send(JSON.stringify(payload));
|
||||
}
|
||||
|
||||
async function ensurePlayback() {
|
||||
if (!playbackContext) {
|
||||
playbackContext = new AudioContext();
|
||||
}
|
||||
await playbackContext.resume();
|
||||
}
|
||||
|
||||
async function playPCM(bytes) {
|
||||
await ensurePlayback();
|
||||
|
||||
const rate = Number($("outputRate").value);
|
||||
const sampleCount = Math.floor(bytes.byteLength / 2);
|
||||
if (!sampleCount) return;
|
||||
|
||||
const view = new DataView(bytes);
|
||||
const samples = new Float32Array(sampleCount);
|
||||
|
||||
for (let i = 0; i < sampleCount; i++) {
|
||||
samples[i] = view.getInt16(i * 2, true) / 32768;
|
||||
}
|
||||
|
||||
const audio = playbackContext.createBuffer(
|
||||
1,
|
||||
sampleCount,
|
||||
rate
|
||||
);
|
||||
audio.copyToChannel(samples, 0);
|
||||
|
||||
const source = playbackContext.createBufferSource();
|
||||
source.buffer = audio;
|
||||
source.connect(playbackContext.destination);
|
||||
|
||||
const startAt = Math.max(
|
||||
playbackContext.currentTime + 0.025,
|
||||
playbackTime
|
||||
);
|
||||
|
||||
source.start(startAt);
|
||||
playbackTime = startAt + audio.duration;
|
||||
|
||||
scheduledAudio.add(source);
|
||||
source.onended = () => scheduledAudio.delete(source);
|
||||
}
|
||||
|
||||
function stopPlayback() {
|
||||
for (const source of scheduledAudio) {
|
||||
try { source.stop(); } catch {}
|
||||
}
|
||||
scheduledAudio.clear();
|
||||
playbackTime = 0;
|
||||
}
|
||||
|
||||
async function stopMicrophone() {
|
||||
if (inputNode) {
|
||||
inputNode.port.onmessage = null;
|
||||
inputNode.disconnect();
|
||||
inputNode = null;
|
||||
}
|
||||
|
||||
inputSource?.disconnect();
|
||||
silentGain?.disconnect();
|
||||
|
||||
inputSource = null;
|
||||
silentGain = null;
|
||||
|
||||
microphone?.getTracks().forEach((track) => track.stop());
|
||||
microphone = null;
|
||||
|
||||
if (inputContext) {
|
||||
const context = inputContext;
|
||||
inputContext = null;
|
||||
await context.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function startMicrophone() {
|
||||
microphone = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true
|
||||
}
|
||||
});
|
||||
|
||||
inputContext = new AudioContext();
|
||||
await inputContext.audioWorklet.addModule("/web/pcm-worklet.js");
|
||||
|
||||
inputSource = inputContext.createMediaStreamSource(microphone);
|
||||
inputNode = new AudioWorkletNode(inputContext, "pcm-input");
|
||||
|
||||
inputNode.port.onmessage = ({ data }) => {
|
||||
if (
|
||||
data.type !== "pcm" ||
|
||||
phase !== "recording" ||
|
||||
turnId === null
|
||||
) return;
|
||||
|
||||
if (ws?.readyState !== WebSocket.OPEN) return;
|
||||
|
||||
sendJson({
|
||||
type: "audio.chunk",
|
||||
turn_id: turnId,
|
||||
sequence_number: sequence++
|
||||
});
|
||||
|
||||
ws.send(data.buffer);
|
||||
};
|
||||
|
||||
silentGain = inputContext.createGain();
|
||||
silentGain.gain.value = 0;
|
||||
|
||||
inputSource.connect(inputNode);
|
||||
inputNode.connect(silentGain);
|
||||
silentGain.connect(inputContext.destination);
|
||||
|
||||
await inputContext.resume();
|
||||
}
|
||||
|
||||
async function beginTurn() {
|
||||
if (phase !== "ready") return;
|
||||
|
||||
try {
|
||||
await ensurePlayback();
|
||||
stopPlayback();
|
||||
|
||||
setPhase("starting");
|
||||
sendJson({ type: "audio.start" });
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
setPhase("ready");
|
||||
}
|
||||
}
|
||||
|
||||
async function endRecording() {
|
||||
if (phase !== "recording") return;
|
||||
|
||||
const activeTurn = turnId;
|
||||
setPhase("processing");
|
||||
|
||||
await stopMicrophone();
|
||||
|
||||
if (activeTurn !== null && ws?.readyState === WebSocket.OPEN) {
|
||||
sendJson({ type: "audio.stop", turn_id: activeTurn });
|
||||
}
|
||||
}
|
||||
|
||||
async function interruptTurn() {
|
||||
if (turnId === null) return;
|
||||
|
||||
const activeTurn = turnId;
|
||||
|
||||
setPhase("processing");
|
||||
await stopMicrophone();
|
||||
stopPlayback();
|
||||
|
||||
if (ws?.readyState === WebSocket.OPEN) {
|
||||
sendJson({ type: "interrupt", turn_id: activeTurn });
|
||||
}
|
||||
}
|
||||
|
||||
async function handleEvent(event) {
|
||||
switch (event.type) {
|
||||
case "session.ready":
|
||||
sessionId = event.session_id;
|
||||
setPhase("ready");
|
||||
break;
|
||||
|
||||
case "turn.started":
|
||||
turnId = event.turn_id;
|
||||
sequence = 0;
|
||||
userMessage = addMessage("user");
|
||||
agentMessage = addMessage("agent");
|
||||
pendingFinal = null;
|
||||
|
||||
try {
|
||||
await startMicrophone();
|
||||
setPhase("recording");
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
talkLabel.textContent = "Não foi possível acessar o microfone.";
|
||||
await interruptTurn();
|
||||
}
|
||||
break;
|
||||
|
||||
case "stt.partial":
|
||||
if (event.turn_id === turnId && userMessage) {
|
||||
userMessage.textContent = event.text;
|
||||
}
|
||||
break;
|
||||
|
||||
case "stt.final":
|
||||
if (event.turn_id === turnId && userMessage) {
|
||||
userMessage.textContent = event.text;
|
||||
}
|
||||
break;
|
||||
|
||||
case "agent.delta":
|
||||
if (event.turn_id === turnId && agentMessage) {
|
||||
agentMessage.textContent += event.text;
|
||||
}
|
||||
break;
|
||||
|
||||
case "agent.response":
|
||||
if (event.turn_id === turnId && agentMessage) {
|
||||
agentMessage.textContent = event.text;
|
||||
}
|
||||
break;
|
||||
|
||||
case "turn.ended":
|
||||
if (event.turn_id === turnId) {
|
||||
turnId = null;
|
||||
await stopMicrophone();
|
||||
setPhase("ready");
|
||||
}
|
||||
break;
|
||||
|
||||
case "interrupted":
|
||||
if (event.turn_id === turnId) {
|
||||
turnId = null;
|
||||
await stopMicrophone();
|
||||
stopPlayback();
|
||||
setPhase("ready");
|
||||
}
|
||||
break;
|
||||
|
||||
case "error":
|
||||
console.error("Gateway error:", event);
|
||||
talkLabel.textContent =
|
||||
`Erro: ${event.code} — ${event.message}`;
|
||||
|
||||
if (phase === "starting") {
|
||||
setPhase("ready");
|
||||
} else if (turnId !== null && phase !== "disconnected") {
|
||||
await interruptTurn();
|
||||
}
|
||||
break;
|
||||
|
||||
case "session.ended":
|
||||
ws?.close();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
async function connect() {
|
||||
if (ws && ws.readyState !== WebSocket.CLOSED) return;
|
||||
|
||||
setPhase("connecting");
|
||||
|
||||
const scheme = location.protocol === "https:" ? "wss" : "ws";
|
||||
ws = new WebSocket(`${scheme}://${location.host}/ws/audio`);
|
||||
ws.binaryType = "arraybuffer";
|
||||
|
||||
// Serialize event handling to preserve ordering.
|
||||
let incoming = Promise.resolve();
|
||||
|
||||
ws.onopen = () => {
|
||||
sendJson({
|
||||
type: "session.start",
|
||||
audio: {
|
||||
encoding: "pcm_s16le",
|
||||
sample_rate: 16000,
|
||||
channels: 1
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
ws.onmessage = ({ data }) => {
|
||||
incoming = incoming.then(async () => {
|
||||
if (data instanceof ArrayBuffer) {
|
||||
if (turnId !== null) {
|
||||
await playPCM(data);
|
||||
}
|
||||
} else {
|
||||
await handleEvent(JSON.parse(data));
|
||||
}
|
||||
}).catch(console.error);
|
||||
};
|
||||
|
||||
ws.onerror = (error) => console.error("WebSocket error", error);
|
||||
|
||||
ws.onclose = async () => {
|
||||
await stopMicrophone();
|
||||
stopPlayback();
|
||||
ws = null;
|
||||
sessionId = null;
|
||||
turnId = null;
|
||||
setPhase("disconnected");
|
||||
};
|
||||
}
|
||||
|
||||
connectButton.addEventListener("click", connect);
|
||||
|
||||
talkButton.addEventListener("click", async () => {
|
||||
if (phase === "ready") await beginTurn();
|
||||
else if (phase === "recording") await endRecording();
|
||||
});
|
||||
|
||||
interruptButton.addEventListener("click", interruptTurn);
|
||||
|
||||
setPhase("disconnected");
|
||||
83
src/voice_assistant/audio_gateway/web/index.html
Normal file
83
src/voice_assistant/audio_gateway/web/index.html
Normal file
@@ -0,0 +1,83 @@
|
||||
|
||||
<!doctype html>
|
||||
<html lang="pt-BR">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport"
|
||||
content="width=device-width, initial-scale=1">
|
||||
<title>AgentVoice · Voice Console</title>
|
||||
<link rel="stylesheet" href="/web/styles.css">
|
||||
</head>
|
||||
<body>
|
||||
<main class="console">
|
||||
<header class="header">
|
||||
<div class="brand">
|
||||
<div class="brand-icon">◉</div>
|
||||
<div>
|
||||
<h1>AgentVoice</h1>
|
||||
<p>Realtime Voice Console</p>
|
||||
</div>
|
||||
</div>
|
||||
<span id="status" class="status">Desconectado</span>
|
||||
</header>
|
||||
|
||||
<section class="hero">
|
||||
<span class="eyebrow">VOICE ASSISTANT · V1</span>
|
||||
<h2>Uma conversa, naturalmente.</h2>
|
||||
<p>Fale pelo microfone e receba respostas
|
||||
geradas por inteligência artificial.</p>
|
||||
</section>
|
||||
|
||||
<section id="messages"
|
||||
class="messages"
|
||||
aria-live="polite">
|
||||
<div class="empty-state" id="emptyState">
|
||||
<div class="empty-icon">◎</div>
|
||||
<p>Sua conversa aparecerá aqui.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="controls">
|
||||
<div class="mic-container">
|
||||
<button id="talkButton"
|
||||
class="talk-button"
|
||||
disabled
|
||||
aria-label="Iniciar gravação">
|
||||
<span id="micIcon">🎙</span>
|
||||
</button>
|
||||
<p id="talkLabel">Conecte para começar</p>
|
||||
</div>
|
||||
|
||||
<div class="actions">
|
||||
<button id="connectButton" class="primary">
|
||||
Conectar
|
||||
</button>
|
||||
<button id="interruptButton"
|
||||
class="secondary" disabled>
|
||||
Interromper
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="settings">
|
||||
<label for="outputRate">
|
||||
Frequência de reprodução PCM
|
||||
</label>
|
||||
<select id="outputRate">
|
||||
<option value="24000">24 kHz</option>
|
||||
<option value="16000">16 kHz</option>
|
||||
<option value="32000">32 kHz</option>
|
||||
<option value="48000">48 kHz</option>
|
||||
</select>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<footer>
|
||||
<span>NVIDIA STT</span>
|
||||
<span>OCI Generative AI</span>
|
||||
<span>VoxCPM TTS</span>
|
||||
</footer>
|
||||
</main>
|
||||
|
||||
<script type="module" src="/web/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
74
src/voice_assistant/audio_gateway/web/pcm-worklet.js
Normal file
74
src/voice_assistant/audio_gateway/web/pcm-worklet.js
Normal file
@@ -0,0 +1,74 @@
|
||||
|
||||
class PCMInputProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.targetRate = 16000;
|
||||
this.step = sampleRate / this.targetRate;
|
||||
this.buffer = [];
|
||||
this.position = 0;
|
||||
this.sequence = 0;
|
||||
this.chunkSamples = 1600;
|
||||
this.output = [];
|
||||
}
|
||||
|
||||
process(inputs) {
|
||||
const channels = inputs[0];
|
||||
|
||||
if (!channels || channels.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const frameLength = channels[0].length;
|
||||
|
||||
for (let i = 0; i < frameLength; i++) {
|
||||
let sample = 0;
|
||||
|
||||
for (const channel of channels) {
|
||||
sample += channel[i];
|
||||
}
|
||||
|
||||
this.buffer.push(sample / channels.length);
|
||||
}
|
||||
|
||||
while (this.position + 1 < this.buffer.length) {
|
||||
const index = Math.floor(this.position);
|
||||
const fraction = this.position - index;
|
||||
|
||||
const sample =
|
||||
this.buffer[index] * (1 - fraction) +
|
||||
this.buffer[index + 1] * fraction;
|
||||
|
||||
const clipped = Math.max(-1, Math.min(1, sample));
|
||||
|
||||
this.output.push(
|
||||
clipped < 0
|
||||
? Math.round(clipped * 32768)
|
||||
: Math.round(clipped * 32767)
|
||||
);
|
||||
|
||||
this.position += this.step;
|
||||
|
||||
if (this.output.length >= this.chunkSamples) {
|
||||
const pcm = new Int16Array(this.output);
|
||||
this.output = [];
|
||||
|
||||
this.port.postMessage({
|
||||
type: "pcm",
|
||||
sequence: this.sequence++,
|
||||
buffer: pcm.buffer
|
||||
}, [pcm.buffer]);
|
||||
}
|
||||
}
|
||||
|
||||
const consumed = Math.floor(this.position);
|
||||
|
||||
if (consumed > 0) {
|
||||
this.buffer.splice(0, consumed);
|
||||
this.position -= consumed;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("pcm-input", PCMInputProcessor);
|
||||
164
src/voice_assistant/audio_gateway/web/styles.css
Normal file
164
src/voice_assistant/audio_gateway/web/styles.css
Normal file
@@ -0,0 +1,164 @@
|
||||
|
||||
:root {
|
||||
font-family: Inter, system-ui, -apple-system, sans-serif;
|
||||
color: #e8edf6;
|
||||
background: #090e18;
|
||||
font-synthesis: none;
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
min-height: 100vh;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
padding: 24px;
|
||||
background:
|
||||
radial-gradient(ellipse at 50% 0%, #17294b, transparent 65%),
|
||||
#090e18;
|
||||
}
|
||||
|
||||
.console {
|
||||
width: min(100%, 820px);
|
||||
background: #111a2a;
|
||||
border: 1px solid #26334a;
|
||||
border-radius: 22px;
|
||||
overflow: hidden;
|
||||
box-shadow: 0 30px 90px #0006;
|
||||
}
|
||||
|
||||
.header, .brand, .actions, footer {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.header {
|
||||
justify-content: space-between;
|
||||
padding: 22px 28px;
|
||||
border-bottom: 1px solid #26334a;
|
||||
}
|
||||
|
||||
.brand { gap: 12px; }
|
||||
.brand-icon { color: #67e8f9; font-size: 34px; }
|
||||
h1 { font-size: 19px; margin: 0; }
|
||||
.brand p, .hero p, .empty-state, footer {
|
||||
color: #93a4bd;
|
||||
}
|
||||
.brand p { margin: 3px 0 0; font-size: 12px; }
|
||||
|
||||
.status {
|
||||
font-size: 12px;
|
||||
border: 1px solid #42506a;
|
||||
border-radius: 20px;
|
||||
padding: 7px 12px;
|
||||
}
|
||||
|
||||
.status.connected {
|
||||
color: #86efac;
|
||||
border-color: #166534;
|
||||
}
|
||||
|
||||
.hero { text-align: center; padding: 36px 24px 24px; }
|
||||
.eyebrow {
|
||||
color: #67e8f9;
|
||||
font-size: 11px;
|
||||
letter-spacing: 2px;
|
||||
}
|
||||
h2 { font-size: clamp(23px, 4vw, 32px); margin: 14px 0; }
|
||||
.hero p { font-size: 14px; line-height: 1.6; }
|
||||
|
||||
.messages {
|
||||
height: 310px;
|
||||
overflow-y: auto;
|
||||
padding: 24px;
|
||||
background: #0c1422;
|
||||
border-block: 1px solid #26334a;
|
||||
}
|
||||
|
||||
.empty-state { text-align: center; padding: 70px 0; }
|
||||
.empty-icon { font-size: 40px; color: #526782; }
|
||||
|
||||
.message {
|
||||
margin: 12px 0;
|
||||
padding: 14px 18px;
|
||||
border-radius: 14px;
|
||||
max-width: 88%;
|
||||
line-height: 1.55;
|
||||
font-size: 14px;
|
||||
white-space: pre-wrap;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.message.user { background: #20385b; margin-left: auto; }
|
||||
.message.agent { background: #1b2b36; margin-right: auto; }
|
||||
.message small {
|
||||
display: block;
|
||||
color: #91a8c5;
|
||||
font-size: 11px;
|
||||
margin-bottom: 5px;
|
||||
}
|
||||
|
||||
.controls { padding: 24px; text-align: center; }
|
||||
.mic-container { padding: 8px 0 20px; }
|
||||
|
||||
.talk-button {
|
||||
width: 88px;
|
||||
height: 88px;
|
||||
border-radius: 50%;
|
||||
border: 1px solid #3a6a9b;
|
||||
background: linear-gradient(145deg, #23598c, #183755);
|
||||
color: white;
|
||||
font-size: 30px;
|
||||
cursor: pointer;
|
||||
transition: transform .15s, box-shadow .15s;
|
||||
}
|
||||
|
||||
.talk-button:hover:not(:disabled) { transform: scale(1.04); }
|
||||
.talk-button.recording {
|
||||
background: #9f2442;
|
||||
border-color: #fb7185;
|
||||
box-shadow: 0 0 0 12px #fb71851a;
|
||||
}
|
||||
|
||||
#talkLabel { color: #a4b4ca; font-size: 13px; }
|
||||
.actions { justify-content: center; gap: 12px; }
|
||||
|
||||
.actions button, select {
|
||||
padding: 11px 20px;
|
||||
border-radius: 9px;
|
||||
border: 1px solid #344862;
|
||||
color: #e8edf6;
|
||||
background: #1b2b42;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.actions .primary { background: #2563eb; border-color: #2563eb; }
|
||||
button:disabled { opacity: .45; cursor: not-allowed; }
|
||||
|
||||
.settings {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 12px;
|
||||
margin-top: 22px;
|
||||
font-size: 12px;
|
||||
color: #93a4bd;
|
||||
}
|
||||
|
||||
footer {
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 24px;
|
||||
border-top: 1px solid #26334a;
|
||||
padding: 16px;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
@media (max-width: 520px) {
|
||||
body { padding: 0; }
|
||||
.console { min-height: 100vh; border-radius: 0; }
|
||||
.header { padding: 18px; }
|
||||
.messages { height: 36vh; }
|
||||
}
|
||||
@@ -41,7 +41,10 @@ import asyncio
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from contextlib import aclosing
|
||||
|
||||
from voice_assistant.agent.text_chunker import TextChunker
|
||||
from voice_assistant.turn_manager import TurnState
|
||||
from voice_assistant.agent.adapter import AgentAdapter
|
||||
from voice_assistant.audio_gateway.app.session import SessionState
|
||||
from voice_assistant.turn_manager import Turn, TurnManager
|
||||
@@ -64,6 +67,21 @@ class PipelineResult:
|
||||
response_text: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PipelineTextDelta:
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PipelineAgentResponse:
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class PipelineAudioChunk:
|
||||
audio: bytes
|
||||
|
||||
|
||||
class VoicePipeline:
|
||||
"""
|
||||
Coordinate session, turn, Agent, and TTS processing.
|
||||
@@ -168,6 +186,184 @@ class VoicePipeline:
|
||||
):
|
||||
yield audio_chunk
|
||||
|
||||
|
||||
async def stream_turn_response(
|
||||
self,
|
||||
session_id: str,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> AsyncIterator[
|
||||
PipelineTextDelta | PipelineAgentResponse | PipelineAudioChunk
|
||||
]:
|
||||
"""
|
||||
Stream Agent text and TTS audio concurrently.
|
||||
|
||||
The producer generates text segments while the consumer
|
||||
synthesizes completed segments in their original order.
|
||||
"""
|
||||
turn = self.turn_manager.active_turn
|
||||
|
||||
if turn is None or turn.turn_id != turn_id:
|
||||
raise ValueError(f"Unexpected turn_id={turn_id}.")
|
||||
|
||||
if turn.state != TurnState.PROCESSING:
|
||||
raise RuntimeError(
|
||||
f"Cannot stream response while turn is {turn.state.value}."
|
||||
)
|
||||
|
||||
stream_method = getattr(self.agent, "stream", None)
|
||||
|
||||
if not callable(stream_method):
|
||||
raise RuntimeError(
|
||||
"Configured Agent does not support streaming."
|
||||
)
|
||||
|
||||
events: asyncio.Queue[
|
||||
PipelineTextDelta | PipelineAgentResponse |
|
||||
PipelineAudioChunk | None
|
||||
] = asyncio.Queue(maxsize=16)
|
||||
|
||||
segments: asyncio.Queue[str | None] = asyncio.Queue(maxsize=4)
|
||||
|
||||
async def produce() -> None:
|
||||
chunker = TextChunker()
|
||||
parts: list[str] = []
|
||||
|
||||
async with aclosing(
|
||||
stream_method(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=text,
|
||||
)
|
||||
) as deltas:
|
||||
async for delta in deltas:
|
||||
parts.append(delta)
|
||||
|
||||
await events.put(
|
||||
PipelineTextDelta(text=delta)
|
||||
)
|
||||
|
||||
for segment in chunker.feed(delta):
|
||||
await segments.put(segment)
|
||||
|
||||
remaining = chunker.flush()
|
||||
|
||||
if remaining:
|
||||
await segments.put(remaining)
|
||||
|
||||
response_text = "".join(parts).strip()
|
||||
|
||||
if not response_text:
|
||||
raise RuntimeError(
|
||||
"Agent returned an empty streaming response."
|
||||
)
|
||||
|
||||
await events.put(
|
||||
PipelineAgentResponse(text=response_text)
|
||||
)
|
||||
|
||||
await segments.put(None)
|
||||
|
||||
async def consume() -> None:
|
||||
first_segment = True
|
||||
|
||||
while True:
|
||||
segment = await segments.get()
|
||||
|
||||
if segment is None:
|
||||
break
|
||||
|
||||
if first_segment:
|
||||
self.turn_manager.on_agent_response(turn_id)
|
||||
first_segment = False
|
||||
|
||||
async with aclosing(
|
||||
self.tts.synthesize(segment)
|
||||
) as audio_stream:
|
||||
async for audio_chunk in audio_stream:
|
||||
await events.put(
|
||||
PipelineAudioChunk(audio=audio_chunk)
|
||||
)
|
||||
|
||||
if first_segment:
|
||||
raise RuntimeError(
|
||||
"No text segments were available for TTS."
|
||||
)
|
||||
|
||||
self.turn_manager.on_tts_complete(turn_id)
|
||||
|
||||
async def run_pipeline() -> None:
|
||||
async with asyncio.TaskGroup() as group:
|
||||
group.create_task(produce())
|
||||
group.create_task(consume())
|
||||
|
||||
execution = asyncio.create_task(run_pipeline())
|
||||
|
||||
pending_get: asyncio.Task | None = None
|
||||
|
||||
try:
|
||||
while True:
|
||||
if execution.done() and events.empty():
|
||||
await execution
|
||||
break
|
||||
|
||||
pending_get = asyncio.create_task(events.get())
|
||||
|
||||
done, _ = await asyncio.wait(
|
||||
{pending_get, execution},
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
|
||||
if pending_get in done:
|
||||
event = pending_get.result()
|
||||
pending_get = None
|
||||
|
||||
if event is not None:
|
||||
yield event
|
||||
|
||||
continue
|
||||
|
||||
# Supervisor completed before another event arrived.
|
||||
# Cancel the pending queue read before inspecting errors.
|
||||
pending_get.cancel()
|
||||
|
||||
await asyncio.gather(
|
||||
pending_get,
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
pending_get = None
|
||||
|
||||
# Propagate failures, including ExceptionGroup.
|
||||
await execution
|
||||
|
||||
# Normal completion: drain remaining events.
|
||||
while not events.empty():
|
||||
event = events.get_nowait()
|
||||
|
||||
if event is not None:
|
||||
yield event
|
||||
|
||||
break
|
||||
|
||||
finally:
|
||||
if pending_get is not None:
|
||||
pending_get.cancel()
|
||||
|
||||
await asyncio.gather(
|
||||
pending_get,
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
if not execution.done():
|
||||
execution.cancel()
|
||||
|
||||
await asyncio.gather(
|
||||
execution,
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
|
||||
def complete_turn(
|
||||
self,
|
||||
turn_id: int,
|
||||
|
||||
Reference in New Issue
Block a user