feat: add OCI GenAI streaming agent foundation
This commit is contained in:
259
src/voice_assistant/agent/oci.py
Normal file
259
src/voice_assistant/agent/oci.py
Normal file
@@ -0,0 +1,259 @@
|
||||
"""
|
||||
OCI Generative AI agent integration.
|
||||
|
||||
Provides parsing and streaming support for OCI Generative AI
|
||||
chat inference using the GenericChatRequest format.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import aclosing
|
||||
|
||||
import oci
|
||||
|
||||
from voice_assistant.agent.adapter import AgentAdapter
|
||||
from voice_assistant.agent.oci_stream import stream_sse_events
|
||||
|
||||
|
||||
def extract_text_delta(raw_data: str) -> str | None:
|
||||
"""
|
||||
Extract incremental text from an OCI Generative AI SSE event.
|
||||
|
||||
Returns:
|
||||
Text delta when available.
|
||||
None for empty or non-text events.
|
||||
|
||||
Raises:
|
||||
ValueError for malformed JSON or invalid event structures.
|
||||
"""
|
||||
if raw_data == "[DONE]":
|
||||
return None
|
||||
|
||||
try:
|
||||
payload: Any = json.loads(raw_data)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(
|
||||
"Invalid JSON in OCI Generative AI SSE event."
|
||||
) from exc
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(
|
||||
"OCI Generative AI SSE event must be an object."
|
||||
)
|
||||
|
||||
message = payload.get("message")
|
||||
|
||||
if message is None:
|
||||
return None
|
||||
|
||||
if not isinstance(message, dict):
|
||||
raise ValueError(
|
||||
"Invalid OCI Generative AI message structure."
|
||||
)
|
||||
|
||||
content = message.get("content", [])
|
||||
|
||||
if not isinstance(content, list):
|
||||
raise ValueError(
|
||||
"Invalid OCI Generative AI content structure."
|
||||
)
|
||||
|
||||
deltas: list[str] = []
|
||||
|
||||
for item in content:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError(
|
||||
"Invalid OCI Generative AI content item."
|
||||
)
|
||||
|
||||
if item.get("type") != "TEXT":
|
||||
continue
|
||||
|
||||
text = item.get("text")
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise ValueError(
|
||||
"Invalid OCI Generative AI text delta."
|
||||
)
|
||||
|
||||
if text:
|
||||
deltas.append(text)
|
||||
|
||||
return "".join(deltas) or None
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OCIAgentAdapter(AgentAdapter):
|
||||
"""
|
||||
OCI Generative AI streaming Agent.
|
||||
|
||||
Responsibilities:
|
||||
- Authenticate using Instance Principal.
|
||||
- Build OCI chat requests.
|
||||
- Execute inference outside the event loop.
|
||||
- Stream text deltas incrementally.
|
||||
- Close SSE resources after consumption.
|
||||
|
||||
Non-responsibilities:
|
||||
- Managing semantic turn state.
|
||||
- Sending WebSocket events.
|
||||
- Performing TTS synthesis.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
region: str,
|
||||
compartment_id: str,
|
||||
model_id: str = "openai.gpt-4o-mini",
|
||||
temperature: float = 0.3,
|
||||
max_completion_tokens: int = 256,
|
||||
timeout_seconds: float = 60.0,
|
||||
) -> None:
|
||||
if not region:
|
||||
raise ValueError("OCI region is required.")
|
||||
|
||||
if not compartment_id:
|
||||
raise ValueError("OCI compartment_id is required.")
|
||||
|
||||
if not model_id:
|
||||
raise ValueError("OCI model_id is required.")
|
||||
|
||||
if timeout_seconds <= 0:
|
||||
raise ValueError("timeout_seconds must be positive.")
|
||||
|
||||
self.region = region
|
||||
self.compartment_id = compartment_id
|
||||
self.model_id = model_id
|
||||
self.temperature = temperature
|
||||
self.max_completion_tokens = max_completion_tokens
|
||||
self.timeout_seconds = timeout_seconds
|
||||
|
||||
signer = (
|
||||
oci.auth.signers
|
||||
.InstancePrincipalsSecurityTokenSigner()
|
||||
)
|
||||
|
||||
self.client = (
|
||||
oci.generative_ai_inference
|
||||
.GenerativeAiInferenceClient(
|
||||
config={"region": region},
|
||||
signer=signer,
|
||||
timeout=(10, timeout_seconds),
|
||||
)
|
||||
)
|
||||
|
||||
def _build_chat_details(
|
||||
self,
|
||||
text: str,
|
||||
):
|
||||
models = oci.generative_ai_inference.models
|
||||
|
||||
chat_request = models.GenericChatRequest(
|
||||
messages=[
|
||||
models.UserMessage(
|
||||
content=[
|
||||
models.TextContent(
|
||||
text=text,
|
||||
)
|
||||
]
|
||||
)
|
||||
],
|
||||
is_stream=True,
|
||||
temperature=self.temperature,
|
||||
max_completion_tokens=self.max_completion_tokens,
|
||||
)
|
||||
|
||||
return models.ChatDetails(
|
||||
compartment_id=self.compartment_id,
|
||||
serving_mode=models.OnDemandServingMode(
|
||||
model_id=self.model_id,
|
||||
),
|
||||
chat_request=chat_request,
|
||||
)
|
||||
|
||||
async def stream(
|
||||
self,
|
||||
session_id: str,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> AsyncIterator[str]:
|
||||
"""
|
||||
Stream OCI Generative AI text deltas.
|
||||
|
||||
The synchronous OCI request is executed outside the
|
||||
event loop. SSE events are consumed through the
|
||||
asynchronous bridge.
|
||||
"""
|
||||
if not text.strip():
|
||||
raise ValueError("Agent input text cannot be empty.")
|
||||
|
||||
details = self._build_chat_details(text)
|
||||
|
||||
logger.debug(
|
||||
"Starting OCI Agent stream. "
|
||||
"session_id=%s turn_id=%s model=%s",
|
||||
session_id,
|
||||
turn_id,
|
||||
self.model_id,
|
||||
)
|
||||
|
||||
response = await asyncio.to_thread(
|
||||
self.client.chat,
|
||||
details,
|
||||
)
|
||||
|
||||
if response.status != 200:
|
||||
raise RuntimeError(
|
||||
f"OCI inference returned HTTP {response.status}."
|
||||
)
|
||||
|
||||
sse = response.data
|
||||
|
||||
async with aclosing(
|
||||
stream_sse_events(sse)
|
||||
) as events:
|
||||
async for raw_data in events:
|
||||
delta = extract_text_delta(raw_data)
|
||||
|
||||
if delta:
|
||||
yield delta
|
||||
|
||||
async def process(
|
||||
self,
|
||||
session_id: str,
|
||||
turn_id: int,
|
||||
text: str,
|
||||
) -> str:
|
||||
"""
|
||||
Collect the complete response for compatibility
|
||||
with the existing AgentAdapter contract.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
|
||||
async with aclosing(
|
||||
self.stream(
|
||||
session_id=session_id,
|
||||
turn_id=turn_id,
|
||||
text=text,
|
||||
)
|
||||
) as deltas:
|
||||
async for delta in deltas:
|
||||
parts.append(delta)
|
||||
|
||||
response_text = "".join(parts).strip()
|
||||
|
||||
if not response_text:
|
||||
raise RuntimeError(
|
||||
"OCI Agent returned an empty response."
|
||||
)
|
||||
|
||||
return response_text
|
||||
|
||||
147
src/voice_assistant/agent/oci_stream.py
Normal file
147
src/voice_assistant/agent/oci_stream.py
Normal file
@@ -0,0 +1,147 @@
|
||||
"""
|
||||
Async bridge for synchronous OCI SSE streams.
|
||||
|
||||
The bridge runs blocking SSE iteration in a worker thread and
|
||||
delivers events to an asynchronous consumer with backpressure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def stream_sse_events(
|
||||
stream: Any,
|
||||
*,
|
||||
max_queue_size: int = 16,
|
||||
shutdown_timeout: float = 5.0,
|
||||
) -> AsyncIterator[str]:
|
||||
"""
|
||||
Convert a synchronous SSE stream into asynchronous text events.
|
||||
|
||||
The stream must provide:
|
||||
events() -> Iterator
|
||||
close() -> None
|
||||
|
||||
Cancellation stops delivery, requests stream closure and
|
||||
waits briefly for the worker to finish.
|
||||
|
||||
Worker termination is not guaranteed if the underlying HTTP
|
||||
read ignores stream closure.
|
||||
"""
|
||||
if max_queue_size < 1:
|
||||
raise ValueError("max_queue_size must be positive.")
|
||||
|
||||
if shutdown_timeout <= 0:
|
||||
raise ValueError("shutdown_timeout must be positive.")
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue(
|
||||
maxsize=max_queue_size,
|
||||
)
|
||||
|
||||
stop_requested = threading.Event()
|
||||
|
||||
def publish(kind: str, value: Any = None) -> bool:
|
||||
if stop_requested.is_set():
|
||||
return False
|
||||
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
queue.put((kind, value)),
|
||||
loop,
|
||||
)
|
||||
|
||||
while not stop_requested.is_set():
|
||||
try:
|
||||
future.result(timeout=0.1)
|
||||
return True
|
||||
except TimeoutError:
|
||||
continue
|
||||
except Exception:
|
||||
future.cancel()
|
||||
raise
|
||||
|
||||
future.cancel()
|
||||
return False
|
||||
|
||||
def worker() -> None:
|
||||
try:
|
||||
for event in stream.events():
|
||||
if stop_requested.is_set():
|
||||
break
|
||||
|
||||
data = getattr(event, "data", None)
|
||||
|
||||
if not isinstance(data, str):
|
||||
continue
|
||||
|
||||
if not publish("data", data):
|
||||
break
|
||||
|
||||
except Exception as exc:
|
||||
if not stop_requested.is_set():
|
||||
publish("error", exc)
|
||||
|
||||
finally:
|
||||
if not stop_requested.is_set():
|
||||
publish("done")
|
||||
|
||||
worker_task = asyncio.create_task(
|
||||
asyncio.to_thread(worker)
|
||||
)
|
||||
|
||||
try:
|
||||
while True:
|
||||
kind, value = await queue.get()
|
||||
|
||||
if kind == "done":
|
||||
break
|
||||
|
||||
if kind == "error":
|
||||
raise value
|
||||
|
||||
yield value
|
||||
|
||||
finally:
|
||||
stop_requested.set()
|
||||
|
||||
# Request transport closure outside the event loop.
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.to_thread(stream.close),
|
||||
timeout=shutdown_timeout,
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.error(
|
||||
"OCI SSE stream closure timed out."
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to close OCI SSE stream."
|
||||
)
|
||||
|
||||
# Do not assume that closing the stream immediately
|
||||
# terminates the synchronous reader.
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(worker_task),
|
||||
timeout=shutdown_timeout,
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.error(
|
||||
"OCI SSE worker did not terminate after closure."
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"OCI SSE worker failed during shutdown."
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user