feat: integrate OCI agent streaming and web voice client

This commit is contained in:
2026-10-13 19:49:27 +00:00
parent 088473a0c8
commit 19a415daae
19 changed files with 2034 additions and 23 deletions

View File

@@ -10,7 +10,6 @@ from __future__ import annotations
import json
from typing import Any
import asyncio
import logging
from collections.abc import AsyncIterator
from contextlib import aclosing
@@ -19,6 +18,7 @@ import oci
from voice_assistant.agent.adapter import AgentAdapter
from voice_assistant.agent.oci_stream import stream_sse_events
from voice_assistant.agent.oci_connection import open_oci_stream
def extract_text_delta(raw_data: str) -> str | None:
@@ -205,9 +205,8 @@ class OCIAgentAdapter(AgentAdapter):
self.model_id,
)
response = await asyncio.to_thread(
self.client.chat,
details,
response = await open_oci_stream(
lambda: self.client.chat(details)
)
if response.status != 200:
@@ -256,4 +255,3 @@ class OCIAgentAdapter(AgentAdapter):
)
return response_text

View File

@@ -0,0 +1,107 @@
"""
Lifecycle management for blocking OCI HTTP connection setup.
Ensures that responses arriving after coroutine cancellation
are closed instead of being abandoned.
"""
from __future__ import annotations
import asyncio
import logging
import threading
from typing import Any, Callable
logger = logging.getLogger(__name__)
def _close_response(response: Any) -> None:
"""Close the SSE response returned by the OCI SDK."""
stream = getattr(response, "data", None)
if stream is not None:
close = getattr(stream, "close", None)
if callable(close):
close()
async def open_oci_stream(
open_connection: Callable[[], Any],
) -> Any:
"""
Open a blocking OCI connection with explicit response ownership.
The worker retains ownership until the async caller claims
the response. Cancellation closes any unclaimed response.
A blocking SDK call may continue after cancellation.
"""
ownership_lock = threading.Lock()
abandoned = False
claimed = False
response = None
def worker() -> Any:
nonlocal response
opened = open_connection()
with ownership_lock:
if abandoned:
should_close = True
else:
response = opened
should_close = False
if should_close:
try:
_close_response(opened)
except Exception:
logger.exception(
"Failed to close abandoned OCI response."
)
return opened
task = asyncio.create_task(
asyncio.to_thread(worker)
)
try:
opened = await asyncio.shield(task)
with ownership_lock:
if abandoned:
raise RuntimeError(
"OCI connection was abandoned."
)
claimed = True
return opened
except asyncio.CancelledError:
with ownership_lock:
abandoned = True
if not claimed:
unclaimed = response
response = None
else:
unclaimed = None
if unclaimed is not None:
try:
await asyncio.to_thread(
_close_response,
unclaimed,
)
except Exception:
logger.exception(
"Failed to close unclaimed OCI response."
)
raise

View File

@@ -0,0 +1,85 @@
"""
Incremental text segmentation for streaming voice synthesis.
Converts LLM text deltas into speech-ready segments while
preserving word boundaries and punctuation.
"""
from __future__ import annotations
class TextChunker:
"""
Accumulate text deltas and emit segments suitable for TTS.
Segments are emitted on sentence-ending punctuation or
at a configurable maximum length.
"""
def __init__(
self,
min_chars: int = 20,
max_chars: int = 160,
) -> None:
if min_chars < 1:
raise ValueError("min_chars must be positive.")
if max_chars < min_chars:
raise ValueError(
"max_chars must be greater than or equal to min_chars."
)
self.min_chars = min_chars
self.max_chars = max_chars
self._buffer = ""
def feed(self, delta: str) -> list[str]:
"""Accept a text delta and return completed segments."""
self._buffer += delta
segments: list[str] = []
while len(self._buffer) >= self.min_chars:
boundary = self._find_boundary()
if boundary is None:
break
segment = self._buffer[:boundary].strip()
self._buffer = self._buffer[boundary:]
if segment:
segments.append(segment)
return segments
def _find_boundary(self) -> int | None:
"""Find a sentence boundary or safe length boundary."""
for index, char in enumerate(self._buffer):
if char in ".!?":
boundary = index + 1
if boundary >= self.min_chars:
return boundary
if len(self._buffer) < self.max_chars:
return None
# Prefer breaking at a word boundary.
boundary = self._buffer.rfind(
" ",
0,
self.max_chars + 1,
)
if boundary >= self.min_chars:
return boundary + 1
return self.max_chars
def flush(self) -> str | None:
"""Return any remaining text after generation completes."""
remaining = self._buffer.strip()
self._buffer = ""
return remaining or None

View File

@@ -34,8 +34,13 @@ Architecture:
import json
import logging
from pathlib import Path
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError
from contextlib import aclosing
import asyncio
@@ -83,8 +88,16 @@ from voice_assistant.audio_gateway.app.protocol import (
STTPartial,
TurnEnded,
TurnStarted,
AgentDelta,
)
from voice_assistant.voice_pipeline import (
PipelineTextDelta,
PipelineAgentResponse,
PipelineAudioChunk,
)
from voice_assistant.audio_gateway.app.session import SessionState
# logger configuration
@@ -100,6 +113,18 @@ provider_catalog = load_provider_catalog()
MAX_AUDIO_CHUNK_BYTES = 64 * 1024 # 64 KB
WEB_DIR = Path(__file__).resolve().parents[1] / "web"
app.mount(
"/web",
StaticFiles(directory=WEB_DIR),
name="web",
)
@app.get("/ui", include_in_schema=False)
async def web_client() -> FileResponse:
return FileResponse(WEB_DIR / "index.html")
def create_turn_manager() -> TurnManager:
@@ -176,27 +201,63 @@ async def forward_stt_events(
),
)
result = await pipeline.process_transcript(
session_id=session_id,
turn_id=turn_id,
text=event.text,
)
if callable(getattr(pipeline.agent, "stream", None)):
async with aclosing(
pipeline.stream_turn_response(
session_id=session_id,
turn_id=turn_id,
text=event.text,
)
) as response_events:
async for response_event in response_events:
if not pipeline.is_active_turn(turn_id):
break
await send_json(
websocket,
AgentResponse(
if isinstance(response_event, PipelineTextDelta):
await send_json(
websocket,
AgentDelta(
session_id=session_id,
turn_id=turn_id,
text=response_event.text,
),
)
elif isinstance(response_event, PipelineAgentResponse):
await send_json(
websocket,
AgentResponse(
session_id=session_id,
turn_id=turn_id,
text=response_event.text,
),
)
elif isinstance(response_event, PipelineAudioChunk):
await websocket.send_bytes(response_event.audio)
else:
result = await pipeline.process_transcript(
session_id=session_id,
turn_id=turn_id,
text=result.response_text,
),
)
text=event.text,
)
async for audio_chunk in pipeline.synthesize_response(
result.response_text,
):
await websocket.send_bytes(audio_chunk)
await send_json(
websocket,
AgentResponse(
session_id=session_id,
turn_id=turn_id,
text=result.response_text,
),
)
pipeline.on_tts_complete(turn_id)
async for audio_chunk in pipeline.synthesize_response(
result.response_text,
):
await websocket.send_bytes(audio_chunk)
pipeline.on_tts_complete(turn_id)

View File

@@ -109,6 +109,20 @@ class STTFinal(BaseModel):
text: str
class AgentDelta(BaseModel):
"""
Incremental text generated by the configured Agent.
Each event belongs to one session and turn.
Text fragments must be concatenated in arrival order.
"""
type: Literal["agent.delta"] = "agent.delta"
session_id: str
turn_id: int
text: str
class AgentResponse(BaseModel):
"""Text response produced by the configured Agent."""

View File

@@ -0,0 +1,387 @@
const $ = (id) => document.getElementById(id);
const status = $("status");
const talkButton = $("talkButton");
const talkLabel = $("talkLabel");
const connectButton = $("connectButton");
const interruptButton = $("interruptButton");
const messages = $("messages");
let ws = null;
let sessionId = null;
let turnId = null;
let sequence = 0;
let phase = "disconnected";
let microphone = null;
let inputContext = null;
let inputSource = null;
let inputNode = null;
let silentGain = null;
let playbackContext = null;
let playbackTime = 0;
const scheduledAudio = new Set();
let userMessage = null;
let agentMessage = null;
let pendingFinal = null;
function setPhase(next) {
phase = next;
status.textContent = {
disconnected: "Desconectado",
connecting: "Conectando",
ready: "Conectado",
starting: "Iniciando turno",
recording: "Gravando",
processing: "Processando",
speaking: "Respondendo"
}[next] || next;
status.classList.toggle(
"connected",
next !== "disconnected" && next !== "connecting"
);
connectButton.disabled = next !== "disconnected";
talkButton.disabled = next !== "ready" && next !== "recording";
interruptButton.disabled = turnId === null;
talkButton.classList.toggle("recording", next === "recording");
talkLabel.textContent = next === "recording"
? "Clique para terminar de falar"
: next === "ready"
? "Clique para falar"
: status.textContent;
}
function addMessage(role, text = "") {
$("emptyState")?.remove();
const element = document.createElement("div");
element.className = `message ${role}`;
const label = document.createElement("small");
label.textContent = role === "user" ? "Você" : "AgentVoice";
const content = document.createElement("span");
content.textContent = text;
element.append(label, content);
messages.appendChild(element);
messages.scrollTop = messages.scrollHeight;
return content;
}
function sendJson(payload) {
if (ws?.readyState !== WebSocket.OPEN) {
throw new Error("WebSocket não está conectado.");
}
ws.send(JSON.stringify(payload));
}
async function ensurePlayback() {
if (!playbackContext) {
playbackContext = new AudioContext();
}
await playbackContext.resume();
}
async function playPCM(bytes) {
await ensurePlayback();
const rate = Number($("outputRate").value);
const sampleCount = Math.floor(bytes.byteLength / 2);
if (!sampleCount) return;
const view = new DataView(bytes);
const samples = new Float32Array(sampleCount);
for (let i = 0; i < sampleCount; i++) {
samples[i] = view.getInt16(i * 2, true) / 32768;
}
const audio = playbackContext.createBuffer(
1,
sampleCount,
rate
);
audio.copyToChannel(samples, 0);
const source = playbackContext.createBufferSource();
source.buffer = audio;
source.connect(playbackContext.destination);
const startAt = Math.max(
playbackContext.currentTime + 0.025,
playbackTime
);
source.start(startAt);
playbackTime = startAt + audio.duration;
scheduledAudio.add(source);
source.onended = () => scheduledAudio.delete(source);
}
function stopPlayback() {
for (const source of scheduledAudio) {
try { source.stop(); } catch {}
}
scheduledAudio.clear();
playbackTime = 0;
}
async function stopMicrophone() {
if (inputNode) {
inputNode.port.onmessage = null;
inputNode.disconnect();
inputNode = null;
}
inputSource?.disconnect();
silentGain?.disconnect();
inputSource = null;
silentGain = null;
microphone?.getTracks().forEach((track) => track.stop());
microphone = null;
if (inputContext) {
const context = inputContext;
inputContext = null;
await context.close();
}
}
async function startMicrophone() {
microphone = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
echoCancellation: true,
noiseSuppression: true,
autoGainControl: true
}
});
inputContext = new AudioContext();
await inputContext.audioWorklet.addModule("/web/pcm-worklet.js");
inputSource = inputContext.createMediaStreamSource(microphone);
inputNode = new AudioWorkletNode(inputContext, "pcm-input");
inputNode.port.onmessage = ({ data }) => {
if (
data.type !== "pcm" ||
phase !== "recording" ||
turnId === null
) return;
if (ws?.readyState !== WebSocket.OPEN) return;
sendJson({
type: "audio.chunk",
turn_id: turnId,
sequence_number: sequence++
});
ws.send(data.buffer);
};
silentGain = inputContext.createGain();
silentGain.gain.value = 0;
inputSource.connect(inputNode);
inputNode.connect(silentGain);
silentGain.connect(inputContext.destination);
await inputContext.resume();
}
async function beginTurn() {
if (phase !== "ready") return;
try {
await ensurePlayback();
stopPlayback();
setPhase("starting");
sendJson({ type: "audio.start" });
} catch (error) {
console.error(error);
setPhase("ready");
}
}
async function endRecording() {
if (phase !== "recording") return;
const activeTurn = turnId;
setPhase("processing");
await stopMicrophone();
if (activeTurn !== null && ws?.readyState === WebSocket.OPEN) {
sendJson({ type: "audio.stop", turn_id: activeTurn });
}
}
async function interruptTurn() {
if (turnId === null) return;
const activeTurn = turnId;
setPhase("processing");
await stopMicrophone();
stopPlayback();
if (ws?.readyState === WebSocket.OPEN) {
sendJson({ type: "interrupt", turn_id: activeTurn });
}
}
async function handleEvent(event) {
switch (event.type) {
case "session.ready":
sessionId = event.session_id;
setPhase("ready");
break;
case "turn.started":
turnId = event.turn_id;
sequence = 0;
userMessage = addMessage("user");
agentMessage = addMessage("agent");
pendingFinal = null;
try {
await startMicrophone();
setPhase("recording");
} catch (error) {
console.error(error);
talkLabel.textContent = "Não foi possível acessar o microfone.";
await interruptTurn();
}
break;
case "stt.partial":
if (event.turn_id === turnId && userMessage) {
userMessage.textContent = event.text;
}
break;
case "stt.final":
if (event.turn_id === turnId && userMessage) {
userMessage.textContent = event.text;
}
break;
case "agent.delta":
if (event.turn_id === turnId && agentMessage) {
agentMessage.textContent += event.text;
}
break;
case "agent.response":
if (event.turn_id === turnId && agentMessage) {
agentMessage.textContent = event.text;
}
break;
case "turn.ended":
if (event.turn_id === turnId) {
turnId = null;
await stopMicrophone();
setPhase("ready");
}
break;
case "interrupted":
if (event.turn_id === turnId) {
turnId = null;
await stopMicrophone();
stopPlayback();
setPhase("ready");
}
break;
case "error":
console.error("Gateway error:", event);
talkLabel.textContent =
`Erro: ${event.code} — ${event.message}`;
if (phase === "starting") {
setPhase("ready");
} else if (turnId !== null && phase !== "disconnected") {
await interruptTurn();
}
break;
case "session.ended":
ws?.close();
break;
}
}
async function connect() {
if (ws && ws.readyState !== WebSocket.CLOSED) return;
setPhase("connecting");
const scheme = location.protocol === "https:" ? "wss" : "ws";
ws = new WebSocket(`${scheme}://${location.host}/ws/audio`);
ws.binaryType = "arraybuffer";
// Serialize event handling to preserve ordering.
let incoming = Promise.resolve();
ws.onopen = () => {
sendJson({
type: "session.start",
audio: {
encoding: "pcm_s16le",
sample_rate: 16000,
channels: 1
}
});
};
ws.onmessage = ({ data }) => {
incoming = incoming.then(async () => {
if (data instanceof ArrayBuffer) {
if (turnId !== null) {
await playPCM(data);
}
} else {
await handleEvent(JSON.parse(data));
}
}).catch(console.error);
};
ws.onerror = (error) => console.error("WebSocket error", error);
ws.onclose = async () => {
await stopMicrophone();
stopPlayback();
ws = null;
sessionId = null;
turnId = null;
setPhase("disconnected");
};
}
connectButton.addEventListener("click", connect);
talkButton.addEventListener("click", async () => {
if (phase === "ready") await beginTurn();
else if (phase === "recording") await endRecording();
});
interruptButton.addEventListener("click", interruptTurn);
setPhase("disconnected");

View File

@@ -0,0 +1,83 @@
<!doctype html>
<html lang="pt-BR">
<head>
<meta charset="utf-8">
<meta name="viewport"
content="width=device-width, initial-scale=1">
<title>AgentVoice · Voice Console</title>
<link rel="stylesheet" href="/web/styles.css">
</head>
<body>
<main class="console">
<header class="header">
<div class="brand">
<div class="brand-icon">◉</div>
<div>
<h1>AgentVoice</h1>
<p>Realtime Voice Console</p>
</div>
</div>
<span id="status" class="status">Desconectado</span>
</header>
<section class="hero">
<span class="eyebrow">VOICE ASSISTANT · V1</span>
<h2>Uma conversa, naturalmente.</h2>
<p>Fale pelo microfone e receba respostas
geradas por inteligência artificial.</p>
</section>
<section id="messages"
class="messages"
aria-live="polite">
<div class="empty-state" id="emptyState">
<div class="empty-icon">◎</div>
<p>Sua conversa aparecerá aqui.</p>
</div>
</section>
<section class="controls">
<div class="mic-container">
<button id="talkButton"
class="talk-button"
disabled
aria-label="Iniciar gravação">
<span id="micIcon">🎙</span>
</button>
<p id="talkLabel">Conecte para começar</p>
</div>
<div class="actions">
<button id="connectButton" class="primary">
Conectar
</button>
<button id="interruptButton"
class="secondary" disabled>
Interromper
</button>
</div>
<div class="settings">
<label for="outputRate">
Frequência de reprodução PCM
</label>
<select id="outputRate">
<option value="24000">24 kHz</option>
<option value="16000">16 kHz</option>
<option value="32000">32 kHz</option>
<option value="48000">48 kHz</option>
</select>
</div>
</section>
<footer>
<span>NVIDIA STT</span>
<span>OCI Generative AI</span>
<span>VoxCPM TTS</span>
</footer>
</main>
<script type="module" src="/web/app.js"></script>
</body>
</html>

View File

@@ -0,0 +1,74 @@
class PCMInputProcessor extends AudioWorkletProcessor {
constructor() {
super();
this.targetRate = 16000;
this.step = sampleRate / this.targetRate;
this.buffer = [];
this.position = 0;
this.sequence = 0;
this.chunkSamples = 1600;
this.output = [];
}
process(inputs) {
const channels = inputs[0];
if (!channels || channels.length === 0) {
return true;
}
const frameLength = channels[0].length;
for (let i = 0; i < frameLength; i++) {
let sample = 0;
for (const channel of channels) {
sample += channel[i];
}
this.buffer.push(sample / channels.length);
}
while (this.position + 1 < this.buffer.length) {
const index = Math.floor(this.position);
const fraction = this.position - index;
const sample =
this.buffer[index] * (1 - fraction) +
this.buffer[index + 1] * fraction;
const clipped = Math.max(-1, Math.min(1, sample));
this.output.push(
clipped < 0
? Math.round(clipped * 32768)
: Math.round(clipped * 32767)
);
this.position += this.step;
if (this.output.length >= this.chunkSamples) {
const pcm = new Int16Array(this.output);
this.output = [];
this.port.postMessage({
type: "pcm",
sequence: this.sequence++,
buffer: pcm.buffer
}, [pcm.buffer]);
}
}
const consumed = Math.floor(this.position);
if (consumed > 0) {
this.buffer.splice(0, consumed);
this.position -= consumed;
}
return true;
}
}
registerProcessor("pcm-input", PCMInputProcessor);

View File

@@ -0,0 +1,164 @@
:root {
font-family: Inter, system-ui, -apple-system, sans-serif;
color: #e8edf6;
background: #090e18;
font-synthesis: none;
}
* { box-sizing: border-box; }
body {
margin: 0;
min-height: 100vh;
display: grid;
place-items: center;
padding: 24px;
background:
radial-gradient(ellipse at 50% 0%, #17294b, transparent 65%),
#090e18;
}
.console {
width: min(100%, 820px);
background: #111a2a;
border: 1px solid #26334a;
border-radius: 22px;
overflow: hidden;
box-shadow: 0 30px 90px #0006;
}
.header, .brand, .actions, footer {
display: flex;
align-items: center;
}
.header {
justify-content: space-between;
padding: 22px 28px;
border-bottom: 1px solid #26334a;
}
.brand { gap: 12px; }
.brand-icon { color: #67e8f9; font-size: 34px; }
h1 { font-size: 19px; margin: 0; }
.brand p, .hero p, .empty-state, footer {
color: #93a4bd;
}
.brand p { margin: 3px 0 0; font-size: 12px; }
.status {
font-size: 12px;
border: 1px solid #42506a;
border-radius: 20px;
padding: 7px 12px;
}
.status.connected {
color: #86efac;
border-color: #166534;
}
.hero { text-align: center; padding: 36px 24px 24px; }
.eyebrow {
color: #67e8f9;
font-size: 11px;
letter-spacing: 2px;
}
h2 { font-size: clamp(23px, 4vw, 32px); margin: 14px 0; }
.hero p { font-size: 14px; line-height: 1.6; }
.messages {
height: 310px;
overflow-y: auto;
padding: 24px;
background: #0c1422;
border-block: 1px solid #26334a;
}
.empty-state { text-align: center; padding: 70px 0; }
.empty-icon { font-size: 40px; color: #526782; }
.message {
margin: 12px 0;
padding: 14px 18px;
border-radius: 14px;
max-width: 88%;
line-height: 1.55;
font-size: 14px;
white-space: pre-wrap;
overflow-wrap: anywhere;
}
.message.user { background: #20385b; margin-left: auto; }
.message.agent { background: #1b2b36; margin-right: auto; }
.message small {
display: block;
color: #91a8c5;
font-size: 11px;
margin-bottom: 5px;
}
.controls { padding: 24px; text-align: center; }
.mic-container { padding: 8px 0 20px; }
.talk-button {
width: 88px;
height: 88px;
border-radius: 50%;
border: 1px solid #3a6a9b;
background: linear-gradient(145deg, #23598c, #183755);
color: white;
font-size: 30px;
cursor: pointer;
transition: transform .15s, box-shadow .15s;
}
.talk-button:hover:not(:disabled) { transform: scale(1.04); }
.talk-button.recording {
background: #9f2442;
border-color: #fb7185;
box-shadow: 0 0 0 12px #fb71851a;
}
#talkLabel { color: #a4b4ca; font-size: 13px; }
.actions { justify-content: center; gap: 12px; }
.actions button, select {
padding: 11px 20px;
border-radius: 9px;
border: 1px solid #344862;
color: #e8edf6;
background: #1b2b42;
cursor: pointer;
}
.actions .primary { background: #2563eb; border-color: #2563eb; }
button:disabled { opacity: .45; cursor: not-allowed; }
.settings {
display: flex;
justify-content: center;
align-items: center;
flex-wrap: wrap;
gap: 12px;
margin-top: 22px;
font-size: 12px;
color: #93a4bd;
}
footer {
justify-content: center;
flex-wrap: wrap;
gap: 24px;
border-top: 1px solid #26334a;
padding: 16px;
font-size: 11px;
}
@media (max-width: 520px) {
body { padding: 0; }
.console { min-height: 100vh; border-radius: 0; }
.header { padding: 18px; }
.messages { height: 36vh; }
}

View File

@@ -41,7 +41,10 @@ import asyncio
from collections.abc import AsyncIterator
from dataclasses import dataclass
from contextlib import aclosing
from voice_assistant.agent.text_chunker import TextChunker
from voice_assistant.turn_manager import TurnState
from voice_assistant.agent.adapter import AgentAdapter
from voice_assistant.audio_gateway.app.session import SessionState
from voice_assistant.turn_manager import Turn, TurnManager
@@ -64,6 +67,21 @@ class PipelineResult:
response_text: str
@dataclass(slots=True)
class PipelineTextDelta:
text: str
@dataclass(slots=True)
class PipelineAgentResponse:
text: str
@dataclass(slots=True)
class PipelineAudioChunk:
audio: bytes
class VoicePipeline:
"""
Coordinate session, turn, Agent, and TTS processing.
@@ -168,6 +186,184 @@ class VoicePipeline:
):
yield audio_chunk
async def stream_turn_response(
self,
session_id: str,
turn_id: int,
text: str,
) -> AsyncIterator[
PipelineTextDelta | PipelineAgentResponse | PipelineAudioChunk
]:
"""
Stream Agent text and TTS audio concurrently.
The producer generates text segments while the consumer
synthesizes completed segments in their original order.
"""
turn = self.turn_manager.active_turn
if turn is None or turn.turn_id != turn_id:
raise ValueError(f"Unexpected turn_id={turn_id}.")
if turn.state != TurnState.PROCESSING:
raise RuntimeError(
f"Cannot stream response while turn is {turn.state.value}."
)
stream_method = getattr(self.agent, "stream", None)
if not callable(stream_method):
raise RuntimeError(
"Configured Agent does not support streaming."
)
events: asyncio.Queue[
PipelineTextDelta | PipelineAgentResponse |
PipelineAudioChunk | None
] = asyncio.Queue(maxsize=16)
segments: asyncio.Queue[str | None] = asyncio.Queue(maxsize=4)
async def produce() -> None:
chunker = TextChunker()
parts: list[str] = []
async with aclosing(
stream_method(
session_id=session_id,
turn_id=turn_id,
text=text,
)
) as deltas:
async for delta in deltas:
parts.append(delta)
await events.put(
PipelineTextDelta(text=delta)
)
for segment in chunker.feed(delta):
await segments.put(segment)
remaining = chunker.flush()
if remaining:
await segments.put(remaining)
response_text = "".join(parts).strip()
if not response_text:
raise RuntimeError(
"Agent returned an empty streaming response."
)
await events.put(
PipelineAgentResponse(text=response_text)
)
await segments.put(None)
async def consume() -> None:
first_segment = True
while True:
segment = await segments.get()
if segment is None:
break
if first_segment:
self.turn_manager.on_agent_response(turn_id)
first_segment = False
async with aclosing(
self.tts.synthesize(segment)
) as audio_stream:
async for audio_chunk in audio_stream:
await events.put(
PipelineAudioChunk(audio=audio_chunk)
)
if first_segment:
raise RuntimeError(
"No text segments were available for TTS."
)
self.turn_manager.on_tts_complete(turn_id)
async def run_pipeline() -> None:
async with asyncio.TaskGroup() as group:
group.create_task(produce())
group.create_task(consume())
execution = asyncio.create_task(run_pipeline())
pending_get: asyncio.Task | None = None
try:
while True:
if execution.done() and events.empty():
await execution
break
pending_get = asyncio.create_task(events.get())
done, _ = await asyncio.wait(
{pending_get, execution},
return_when=asyncio.FIRST_COMPLETED,
)
if pending_get in done:
event = pending_get.result()
pending_get = None
if event is not None:
yield event
continue
# Supervisor completed before another event arrived.
# Cancel the pending queue read before inspecting errors.
pending_get.cancel()
await asyncio.gather(
pending_get,
return_exceptions=True,
)
pending_get = None
# Propagate failures, including ExceptionGroup.
await execution
# Normal completion: drain remaining events.
while not events.empty():
event = events.get_nowait()
if event is not None:
yield event
break
finally:
if pending_get is not None:
pending_get.cancel()
await asyncio.gather(
pending_get,
return_exceptions=True,
)
if not execution.done():
execution.cancel()
await asyncio.gather(
execution,
return_exceptions=True,
)
def complete_turn(
self,
turn_id: int,