adding DB Error in Observability

This commit is contained in:
T3782834
2026-10-28 13:05:53 -03:00
parent f8197334a2
commit e517f5e6a3
9 changed files with 134 additions and 39 deletions

View File

@@ -236,8 +236,10 @@ class MCPToolRouter:
mcp_server=("mcp_gateway" if self.gateway_enabled else server.name),
input=mapped_arguments,
tags=["mcp", "tool", "mcp_gateway" if self.gateway_enabled else "mcp_server"],
):
) as span:
result = await _execute()
if not result.ok:
span.set_status("ERROR", result.error or "MCP tool failed")
await self.telemetry.event(
"mcp.tool_call.completed",
{

View File

@@ -33,7 +33,9 @@ class LongTermMemoryManager:
return []
try:
return await self.store.search(tenant_id=tenant, agent_id=agent, subject_key=subject, limit=int(getattr(self.settings, 'LONG_TERM_MEMORY_MAX_CONTEXT_ITEMS', 20)))
except Exception:
except Exception as exc:
if self.telemetry:
await self.telemetry.event('memory.long_term.load.failed', {'error': str(exc), 'fallback': True}, kind='memory')
logger.exception('Falha não crítica ao carregar LTM')
return []
@@ -49,6 +51,8 @@ class LongTermMemoryManager:
saved = await self.store.upsert_many(tenant_id=tenant, agent_id=agent, subject_key=subject, items=candidates, source_session_id=str(state.get('conversation_key') or state.get('session_id') or ''), source_message_id=str((state.get('context') or {}).get('message_id') or ''))
return {'saved': len(saved), 'items': [item.to_dict() for item in saved]}
except Exception as exc:
if self.telemetry:
await self.telemetry.event('memory.long_term.persist.failed', {'error': str(exc)}, kind='memory')
logger.exception('Falha não crítica ao persistir LTM')
return {'saved': 0, 'error': str(exc)}

View File

@@ -10,4 +10,4 @@ class GuardrailTelemetry:
async def blocked(self, stage: str, decision: Any):
payload = decision.model_dump() if hasattr(decision, "model_dump") else dict(decision or {})
payload.update({"stage": stage})
await self.telemetry.event(f"guardrail.{payload.get('code', 'unknown')}.blocked", payload, kind="guardrail")
await self.telemetry.event(f"guardrail.{payload.get('code', 'unknown')}.blocked", payload, kind="guardrail", level="WARNING")

View File

@@ -6,4 +6,4 @@ class JudgeTelemetry:
async def evaluated(self, result: Any, latency_ms: int | None = None):
payload = result.model_dump() if hasattr(result, "model_dump") else dict(result or {})
payload.update({"latency_ms": latency_ms})
await self.telemetry.event(f"judge.{payload.get('name', 'unknown')}.evaluated", payload, kind="judge")
await self.telemetry.event(f"judge.{payload.get('name', 'unknown')}.evaluated", payload, kind="judge", level="WARNING" if any(payload.get(k) is False for k in ("approved", "passed", "accepted")) else None)

View File

@@ -0,0 +1,30 @@
"""Conservative operational severity policy for Langfuse observations.
Only failures explicitly evidenced by event names/payloads are elevated.
Normal cache misses, empty memory reads and normal retrievals are not warnings.
"""
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class Severity:
level: str
message: str | None = None
def classify_event(name: str, payload: dict[str, Any], *, explicit_level: str | None = None, explicit_message: str | None = None) -> Severity:
name = str(name or "").lower()
payload = payload or {}
message = explicit_message or str(payload.get("error") or payload.get("reason") or "") or None
if explicit_level is not None:
return Severity(explicit_level, message)
if name.startswith("guardrail.") and name.endswith(".blocked"):
return Severity("WARNING", message or "Guardrail blocked the request or response")
if name.startswith("judge.") and (payload.get("approved") is False or payload.get("passed") is False or payload.get("accepted") is False):
return Severity("WARNING", message or "Judge rejected the response")
if name.startswith("mcp.") and payload.get("ok") is False:
return Severity("ERROR", message or "MCP tool failed")
if name.startswith("rag.") and (name.endswith(".failed") or payload.get("error")):
return Severity("WARNING" if name.endswith(("rewrite.failed", "compress.failed")) else "ERROR", message or name)
if name.startswith(("memory.", "cache.", "checkpoint.")) and (name.endswith(".failed") or payload.get("error")):
return Severity("WARNING" if payload.get("fallback") else "ERROR", message or name)
return Severity("DEFAULT")

View File

@@ -206,6 +206,15 @@ class _SpanHandle:
self.output: Any = None
self.has_output = False
self.metadata: dict[str, Any] = {}
self.level: str | None = None
self.status_message: str | None = None
def set_status(self, level: str, message: str | None = None) -> None:
"""Mark an operational failure without raising an exception."""
if level not in {"DEFAULT", "WARNING", "ERROR", "DEBUG"}:
raise ValueError(f"Invalid Langfuse level: {level}")
self.level = level
self.status_message = message
def set_observation(self, observation: Any | None) -> None:
self.observation = observation
@@ -442,22 +451,26 @@ class Telemetry:
await self.event_bus.publish(f"{name}.started", attrs, kind="span")
yield handle
duration_ms = int((time.time() - start) * 1000)
status = {"status": "ok", "duration_ms": duration_ms}
status = {"status": "error" if handle.level == "ERROR" else "degraded" if handle.level == "WARNING" else "ok", "duration_ms": duration_ms}
out = handle.output if handle.has_output else status
metadata = {**observation_metadata, **status, **handle.metadata}
if span_events is not None:
metadata["aggregated_event_count"] = len(span_events)
metadata["aggregated_events"] = span_events
self._update_observation(observation, input=attrs.get("input"), output=out, metadata=metadata)
self._update_observation(observation, input=attrs.get("input"), output=out, metadata=metadata, level=handle.level, status_message=handle.status_message)
if is_root_span:
self._set_trace_io(observation, input=attrs.get("input"), output=out)
legacy_io_update = {
"input": attrs.get("input"),
"output": out,
"metadata": metadata,
"level": handle.level,
"status_message": handle.status_message,
}
if otel_span is not None:
otel_span.set_attribute("duration_ms", duration_ms)
if handle.level in ("WARNING", "ERROR"):
otel_span.set_attribute("operational.level", handle.level)
completed_payload = {**attrs, **status}
if handle.has_output:
completed_payload["output"] = out
@@ -510,7 +523,11 @@ class Telemetry:
try: otel_cm.__exit__(None, None, None)
except Exception: logger.debug("Falha ao fechar span OTEL", exc_info=True)
async def event(self, name: str, payload: dict[str, Any] | None = None, *, kind: str = "event"):
async def event(self, name: str, payload: dict[str, Any] | None = None, *, kind: str = "event", level: str | None = None, status_message: str | None = None):
from .severity_policy import classify_event
if level is not None and level not in {"DEBUG", "DEFAULT", "WARNING", "ERROR"}:
raise ValueError(f"Invalid Langfuse level: {level}")
severity = classify_event(name, payload or {}, explicit_level=level, explicit_message=status_message)
name, payload, mapping_metadata = self.code_mapper.normalize_payload(name, payload, None)
if mapping_metadata:
payload = {**payload, **mapping_metadata}
@@ -524,14 +541,14 @@ class Telemetry:
"kind": kind,
"payload": payload,
})
if not _is_compact_visible_event(name) or not self.is_enabled():
if (not _is_compact_visible_event(name) and severity.level not in ("WARNING", "ERROR")) or not self.is_enabled():
return
try:
metadata = {**payload, "event_kind": kind}
cm = self._start_observation(name=name, as_type="span", input=payload, metadata=metadata)
cm = self._start_observation(name=name, as_type="span", input=payload, metadata=metadata, level=severity.level, status_message=severity.message)
if cm is not None:
with cm as obs:
self._update_observation(obs, input=payload, output={"status": "ok"}, metadata=metadata)
self._update_observation(obs, input=payload, output={"status": "error" if severity.level == "ERROR" else "warning" if severity.level == "WARNING" else "ok"}, metadata=metadata, level=severity.level, status_message=severity.message)
except Exception:
logger.exception("Falha ao enviar event compacto via observation")
return
@@ -545,7 +562,7 @@ class Telemetry:
metadata = {**payload, "event_kind": kind}
if self.is_compact_mode():
metadata["_ignore_current_parent"] = True
cm = self._start_observation(name=name, as_type=_langfuse_type(kind), metadata=metadata)
cm = self._start_observation(name=name, as_type=_langfuse_type(kind), metadata=metadata, level=severity.level, status_message=severity.message)
if cm is not None:
with cm: pass
except Exception: