new feature: JEV Integration. Now there are a current (native) LLM decision model and a JEV decision model
This commit is contained in:
@@ -24,3 +24,5 @@ Requires-Dist: motor>=3.6.0
|
||||
Requires-Dist: google-cloud-pubsub>=2.28.0
|
||||
Requires-Dist: mcp>=1.9.0
|
||||
Requires-Dist: PyJWT[crypto]>=2.9.0
|
||||
Provides-Extra: a2a
|
||||
Requires-Dist: a2a-sdk[http-server]==1.2.2; extra == "a2a"
|
||||
|
||||
@@ -3,6 +3,7 @@ src/agent_framework/__init__.py
|
||||
src/agent_framework/extensions.py
|
||||
src/agent_framework/gateway_policy_context.py
|
||||
src/agent_framework/idempotency.py
|
||||
src/agent_framework/localization.py
|
||||
src/agent_framework/observer.py
|
||||
src/agent_framework/runtime_mcp_gateway_adapter.py
|
||||
src/agent_framework.egg-info/PKG-INFO
|
||||
@@ -10,6 +11,10 @@ src/agent_framework.egg-info/SOURCES.txt
|
||||
src/agent_framework.egg-info/dependency_links.txt
|
||||
src/agent_framework.egg-info/requires.txt
|
||||
src/agent_framework.egg-info/top_level.txt
|
||||
src/agent_framework/a2a/__init__.py
|
||||
src/agent_framework/a2a/client.py
|
||||
src/agent_framework/a2a/progress.py
|
||||
src/agent_framework/a2a/server.py
|
||||
src/agent_framework/analytics/__init__.py
|
||||
src/agent_framework/analytics/composite_publisher.py
|
||||
src/agent_framework/analytics/event_builder.py
|
||||
@@ -39,6 +44,10 @@ src/agent_framework/config/__init__.py
|
||||
src/agent_framework/config/agent_registry.py
|
||||
src/agent_framework/config/observability_mapping.yaml
|
||||
src/agent_framework/config/settings.py
|
||||
src/agent_framework/decisions/__init__.py
|
||||
src/agent_framework/decisions/client.py
|
||||
src/agent_framework/decisions/adapters/__init__.py
|
||||
src/agent_framework/decisions/adapters/typesafe.py
|
||||
src/agent_framework/events/__init__.py
|
||||
src/agent_framework/events/oci_streaming.py
|
||||
src/agent_framework/gateways/__init__.py
|
||||
@@ -138,6 +147,8 @@ src/agent_framework/llm/profile_resolver.py
|
||||
src/agent_framework/llm/providers.py
|
||||
src/agent_framework/llm/structured_output.py
|
||||
src/agent_framework/llm/types.py
|
||||
src/agent_framework/locales/en.json
|
||||
src/agent_framework/locales/pt-BR.json
|
||||
src/agent_framework/mcp/__init__.py
|
||||
src/agent_framework/mcp/client.py
|
||||
src/agent_framework/mcp/models.py
|
||||
@@ -174,6 +185,7 @@ src/agent_framework/observability/noc_events.py
|
||||
src/agent_framework/observability/noc_otel.py
|
||||
src/agent_framework/observability/observer.py
|
||||
src/agent_framework/observability/otel.py
|
||||
src/agent_framework/observability/severity_policy.py
|
||||
src/agent_framework/observability/streaming_events.py
|
||||
src/agent_framework/observability/streaming_exporter.py
|
||||
src/agent_framework/observability/telemetry.py
|
||||
|
||||
@@ -19,3 +19,6 @@ motor>=3.6.0
|
||||
google-cloud-pubsub>=2.28.0
|
||||
mcp>=1.9.0
|
||||
PyJWT[crypto]>=2.9.0
|
||||
|
||||
[a2a]
|
||||
a2a-sdk[http-server]==1.2.2
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from agent_framework.decisions import invoke as decision_invoke, predicate
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
@@ -55,8 +57,11 @@ async def classify_processing_interruption(
|
||||
f"Complemento/interrupção atual: {supplement_client}\n"
|
||||
)
|
||||
try:
|
||||
response = await llm.ainvoke(
|
||||
response = await decision_invoke(llm,
|
||||
[{"role": "system", "content": prompt}],
|
||||
decision_point="processing_interruption",
|
||||
decision_questions=[predicate("regenerate", "Does the interruption require regenerating the response?")],
|
||||
decision_render=lambda d: "1" if d["regenerate"] else "0",
|
||||
temperature=0,
|
||||
fallback_max_tokens=8,
|
||||
profile_name=profile_name,
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,2 @@
|
||||
"""Opt-in typed decision providers. Generative tasks keep their existing provider."""
|
||||
from .client import choice, predicate, score, invoke, selected, evaluate
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
"""Optional decision-provider adapters; no changes to the default engine."""
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,97 @@
|
||||
"""Isolated TypeSafe API adapter. Uses the verified `noul` wire primitive.
|
||||
|
||||
Choice is implemented as independent binary evaluations with strict rejection of
|
||||
ambiguous outcomes. This is an adapter strategy, NOT a native TypeSafe choice
|
||||
contract. The framework's original decision engine is never changed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import os
|
||||
import logging
|
||||
import json
|
||||
|
||||
logger = logging.getLogger("agent_framework.decisions")
|
||||
|
||||
class TypeSafeContractError(ValueError):
|
||||
def __init__(self, code: str):
|
||||
self.code = code
|
||||
super().__init__(code)
|
||||
|
||||
|
||||
def _wire_name(index: int, option: int | None = None) -> str:
|
||||
return f"__jev_q{index}" if option is None else f"__jev_q{index}_o{option}"
|
||||
|
||||
|
||||
def build_request(state: str, questions: list[dict]) -> dict:
|
||||
converted = {}
|
||||
for i, question in enumerate(questions):
|
||||
kind = question["type"]
|
||||
if kind == "predicate":
|
||||
converted[_wire_name(i)] = {"type": "noul", "instructions": question["instructions"]}
|
||||
elif kind == "choice":
|
||||
options = question["choices"]
|
||||
if not options:
|
||||
raise TypeSafeContractError("EMPTY_CHOICE")
|
||||
for j, option in enumerate(options):
|
||||
# Do not interpolate the user's state into the instruction.
|
||||
converted[_wire_name(i, j)] = {
|
||||
"type": "noul",
|
||||
"instructions": (
|
||||
f"Assess whether '{option['value']}' is the correct selection for "
|
||||
f"the decision described here: {question['instructions']}. "
|
||||
"Evaluate the supplied state and policy; answer yes only if this "
|
||||
"specific option is appropriate."
|
||||
),
|
||||
}
|
||||
elif kind == "score":
|
||||
# The direct API score contract has not been verified. Fail closed.
|
||||
raise TypeSafeContractError("UNSUPPORTED_DIRECT_SCORE")
|
||||
else:
|
||||
raise TypeSafeContractError("UNSUPPORTED_QUESTION_TYPE")
|
||||
return {"model": os.getenv("JEV_MODEL", "jev-latest"), "state": state, "questions": converted}
|
||||
|
||||
|
||||
def _probability(answer: object) -> float:
|
||||
if not isinstance(answer, dict) or answer.get("type") not in (None, "noul", "predicate"):
|
||||
raise TypeSafeContractError("INVALID_NOUL_ANSWER")
|
||||
value = answer.get("noul", answer.get("probability"))
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 <= value <= 1:
|
||||
raise TypeSafeContractError("INVALID_NOUL_PROBABILITY")
|
||||
return float(value)
|
||||
|
||||
|
||||
def normalize_answers(response: dict, questions: list[dict]) -> list[dict]:
|
||||
if not isinstance(response, dict) or not isinstance(response.get("answers"), dict):
|
||||
raise TypeSafeContractError("INVALID_ANSWERS_CONTAINER")
|
||||
answers = response["answers"]
|
||||
expected = set()
|
||||
for i, q in enumerate(questions):
|
||||
if q["type"] == "choice":
|
||||
expected.update(_wire_name(i, j) for j in range(len(q["choices"])))
|
||||
else:
|
||||
expected.add(_wire_name(i))
|
||||
if set(answers) != expected:
|
||||
raise TypeSafeContractError("MISSING_OR_EXTRA_ANSWER")
|
||||
result = []
|
||||
for i, q in enumerate(questions):
|
||||
kind = q["type"]
|
||||
if kind == "predicate":
|
||||
result.append({"name": q["name"], "type": "predicate", "probability": _probability(answers[_wire_name(i)])})
|
||||
elif kind == "choice":
|
||||
probabilities = [_probability(answers[_wire_name(i, j)]) for j in range(len(q["choices"]))]
|
||||
ranked = sorted(range(len(probabilities)), key=lambda j: probabilities[j], reverse=True)
|
||||
winner, runner_up = probabilities[ranked[0]], probabilities[ranked[1]] if len(ranked) > 1 else 0.0
|
||||
# Require a single positive, unambiguous choice. Low confidence
|
||||
# remains subject to the unchanged JEV_MIN_CONFIDENCE in client.py.
|
||||
if os.getenv("JEV_DECISION_DIAGNOSTICS", "false").strip().lower() == "true":
|
||||
logger.info("jev.choice.diagnostics question_index=%d probabilities=%s winner_index=%d winner=%.4f runner_up=%.4f margin=%.4f status=%s",
|
||||
i, json.dumps([round(v, 4) for v in probabilities]), ranked[0],
|
||||
winner, runner_up, winner - runner_up,
|
||||
"ambiguous" if winner <= 0.5 or winner - runner_up < 0.15 else "candidate")
|
||||
if winner <= 0.5 or winner - runner_up < 0.15:
|
||||
raise TypeSafeContractError("AMBIGUOUS_DIRECT_CHOICE")
|
||||
result.append({"name": q["name"], "type": "choice",
|
||||
"choice": q["choices"][ranked[0]]["value"], "confidence": winner,
|
||||
"option_probabilities": probabilities})
|
||||
else:
|
||||
raise TypeSafeContractError("UNSUPPORTED_DIRECT_QUESTION_TYPE")
|
||||
return result
|
||||
298
libs/agent_framework/src/agent_framework/decisions/client.py
Normal file
298
libs/agent_framework/src/agent_framework/decisions/client.py
Normal file
@@ -0,0 +1,298 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("agent_framework.decisions")
|
||||
|
||||
|
||||
def env_key(point: str) -> str:
|
||||
return "DECISION_" + re.sub(r"[^A-Z0-9]+", "_", point.upper()).strip("_") + "_PROVIDER"
|
||||
|
||||
|
||||
def selected(point: str, group: str | None = None) -> bool:
|
||||
default = os.getenv(env_key(group), os.getenv("DECISION_PROVIDER", "current")) if group else os.getenv("DECISION_PROVIDER", "current")
|
||||
value = os.getenv(env_key(point), default).strip().lower()
|
||||
if value not in {"current", "jev"}:
|
||||
raise ValueError(f"{env_key(point)} must be current or jev")
|
||||
return value == "jev"
|
||||
|
||||
|
||||
def choice(name: str, values: list, instructions: str = "") -> dict:
|
||||
values = list(dict.fromkeys(str(v) for v in values))
|
||||
return {"type": "choice", "name": name, "instructions": instructions or f"Select {name} following the supplied policy.",
|
||||
"choices": [{"value": v, "description": v} for v in values]}
|
||||
|
||||
|
||||
def predicate(name: str, instructions: str = "") -> dict:
|
||||
return {"type": "predicate", "name": name, "instructions": instructions or f"Is {name} true under the supplied policy?"}
|
||||
|
||||
|
||||
def score(name: str, instructions: str = "") -> dict:
|
||||
return {"type": "score", "name": name, "instructions": instructions or "Apply the supplied rubric on a 0 to 10 scale.",
|
||||
"levels": [{"label": str(i)} for i in range(11)]}
|
||||
|
||||
|
||||
def _number(value: Any, low: float, high: float) -> float:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or not low <= value <= high:
|
||||
raise _validation_error("INVALID_PROBABILITY_OR_SCORE")
|
||||
return float(value)
|
||||
|
||||
|
||||
def _jev_endpoint() -> str:
|
||||
"""Return a trusted HTTPS decision endpoint configured by the deployer.
|
||||
|
||||
The endpoint is an operator setting, never derived from model output.
|
||||
Set JEV_ALLOWED_HOSTS in production to restrict credential destinations.
|
||||
"""
|
||||
endpoint = os.getenv("JEV_ENDPOINT", "https://ai-gateway.vercel.sh/v1/decisions").strip()
|
||||
parsed = urllib.parse.urlsplit(endpoint)
|
||||
if (parsed.scheme != "https" or not parsed.hostname or parsed.username or
|
||||
parsed.password or parsed.fragment or not parsed.path):
|
||||
raise ValueError("JEV_ENDPOINT must be a valid HTTPS endpoint URL")
|
||||
allowed = [host.strip().lower() for host in os.getenv(
|
||||
"JEV_ALLOWED_HOSTS", "ai-gateway.vercel.sh,api.typesafe.ai").split(",") if host.strip()]
|
||||
if parsed.hostname.lower() not in allowed:
|
||||
raise ValueError("JEV_ENDPOINT host is not in JEV_ALLOWED_HOSTS")
|
||||
return endpoint
|
||||
|
||||
|
||||
def _post(body: dict) -> dict:
|
||||
key = (os.getenv("JEV_API_KEY") or os.getenv("AI_GATEWAY_API_KEY") or "").strip()
|
||||
if not key:
|
||||
raise ValueError("JEV_API_KEY or AI_GATEWAY_API_KEY is required for JEV")
|
||||
timeout = float(os.getenv("JEV_TIMEOUT_SECONDS", "15"))
|
||||
if not math.isfinite(timeout) or timeout <= 0:
|
||||
raise ValueError("JEV_TIMEOUT_SECONDS must be positive")
|
||||
request = urllib.request.Request(_jev_endpoint(),
|
||||
data=json.dumps(body, ensure_ascii=False).encode("utf-8"), method="POST",
|
||||
headers={"Authorization": "Bearer " + key, "Content-Type": "application/json"})
|
||||
# urllib does not automatically follow redirects for POST in all cases;
|
||||
# disable redirects to avoid sending credentials to another host.
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
return None
|
||||
|
||||
opener = urllib.request.build_opener(NoRedirect)
|
||||
with opener.open(request, timeout=timeout) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def _safe_http_diagnostic(exc: urllib.error.HTTPError) -> str:
|
||||
"""Extract only structural validation hints; never log echoed inputs or tokens."""
|
||||
if exc.code not in (400, 422):
|
||||
return "unavailable"
|
||||
try:
|
||||
raw = exc.read(16384)
|
||||
payload = json.loads(raw)
|
||||
except (ValueError, TypeError, OSError, UnicodeError):
|
||||
return "unavailable"
|
||||
if not isinstance(payload, dict):
|
||||
return "unavailable"
|
||||
detail = payload.get("detail")
|
||||
if not isinstance(detail, list):
|
||||
return "unavailable"
|
||||
safe = []
|
||||
for item in detail[:5]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
# Only the Pydantic error *type* is safe to expose, not message/input/context.
|
||||
kind = item.get("type")
|
||||
if not isinstance(kind, str) or not re.fullmatch(r"[a-z][a-z0-9_]{0,63}", kind):
|
||||
continue
|
||||
loc = item.get("loc")
|
||||
# Restrict locations to known schema tokens; never echo dynamic names.
|
||||
known = {"body", "model", "state", "questions", "type", "instructions",
|
||||
"choices", "levels", "value", "description", "label"}
|
||||
path = ".".join(part for part in loc if isinstance(part, str) and part in known) if isinstance(loc, list) else ""
|
||||
safe.append(kind + (":" + path if path else ""))
|
||||
return ",".join(safe) or "unavailable"
|
||||
|
||||
|
||||
def _diagnostics_enabled() -> bool:
|
||||
"""Opt-in numeric decision diagnostics; never expose state, prompts or choice text."""
|
||||
return os.getenv("JEV_DECISION_DIAGNOSTICS", "false").strip().lower() == "true"
|
||||
|
||||
|
||||
def _validation_error(code: str) -> ValueError:
|
||||
# Explicit, stable codes for logs; existing fallback/exception behavior stays unchanged.
|
||||
return ValueError(code)
|
||||
|
||||
|
||||
async def evaluate(point: str, messages: list, questions: list[dict], *, llm=None, operation=None, group=None) -> dict:
|
||||
if not questions or len({q["name"] for q in questions}) != len(questions):
|
||||
raise ValueError("Decision questions must have unique names")
|
||||
if any(q["type"] == "choice" and not q["choices"] for q in questions):
|
||||
raise ValueError("Decision choice requires options")
|
||||
started = time.monotonic()
|
||||
state = json.dumps({"policy_and_context": messages}, ensure_ascii=False, default=str)
|
||||
# The default Vercel wire contract remains byte-for-byte equivalent.
|
||||
direct_typesafe = urllib.parse.urlsplit(_jev_endpoint()).hostname == "api.typesafe.ai"
|
||||
if direct_typesafe:
|
||||
from .adapters.typesafe import build_request, normalize_answers
|
||||
request_body = build_request(state, questions)
|
||||
else:
|
||||
request_body = {"model": os.getenv("JEV_MODEL", "typesafe-ai/jev"),
|
||||
"input": state, "questions": questions}
|
||||
response = await asyncio.to_thread(_post, request_body)
|
||||
await _record(llm, point, response, operation)
|
||||
answers = normalize_answers(response, questions) if direct_typesafe else response.get("answers")
|
||||
if not isinstance(answers, list) or len(answers) != len(questions):
|
||||
raise _validation_error("MISSING_DECISION_ANSWERS")
|
||||
indexed = {a.get("name"): a for a in answers if isinstance(a, dict)}
|
||||
if len(indexed) != len(questions):
|
||||
raise _validation_error("DUPLICATE_OR_INVALID_ANSWERS")
|
||||
result, confidences, predicate_probabilities = {}, [], []
|
||||
diagnostic_questions = []
|
||||
for q in questions:
|
||||
a = indexed.get(q["name"], {})
|
||||
if a.get("type") != q["type"]:
|
||||
raise _validation_error("ANSWER_TYPE_MISMATCH")
|
||||
if q["type"] == "choice":
|
||||
value = a.get("choice")
|
||||
if value not in {v["value"] for v in q["choices"]}:
|
||||
raise _validation_error("UNDECLARED_CHOICE")
|
||||
# Missing confidence must never be promoted to certainty.
|
||||
confidences.append(_number(a.get("confidence", 0), 0, 1))
|
||||
if _diagnostics_enabled():
|
||||
options = [v["value"] for v in q["choices"]]
|
||||
detail = {"question_index": len(diagnostic_questions), "type": "choice",
|
||||
"selected_option_index": options.index(value),
|
||||
"confidence": round(confidences[-1], 4)}
|
||||
if isinstance(a.get("option_probabilities"), list):
|
||||
detail["option_probabilities"] = [round(_number(v, 0, 1), 4) for v in a["option_probabilities"]]
|
||||
diagnostic_questions.append(detail)
|
||||
elif q["type"] == "predicate":
|
||||
probability = _number(a.get("probability"), 0, 1)
|
||||
value = probability >= 0.5
|
||||
confidences.append(max(probability, 1-probability))
|
||||
predicate_probabilities.append((q["name"], probability))
|
||||
if _diagnostics_enabled():
|
||||
diagnostic_questions.append({"question_index": len(diagnostic_questions), "type": "predicate",
|
||||
"decision": value, "probability_true": round(probability, 4),
|
||||
"confidence": round(confidences[-1], 4)})
|
||||
else:
|
||||
value = _number(a.get("score"), 0, len(q["levels"])-1)
|
||||
confidences.append(_number(a.get("confidence", 0), 0, 1))
|
||||
if _diagnostics_enabled():
|
||||
diagnostic_questions.append({"question_index": len(diagnostic_questions), "type": "score",
|
||||
"score": value, "confidence": round(confidences[-1], 4)})
|
||||
result[q["name"]] = value
|
||||
confidence = min(confidences)
|
||||
default_minimum = os.getenv(env_key(group).replace("_PROVIDER", "_MIN_CONFIDENCE"), os.getenv("JEV_MIN_CONFIDENCE", "0.7")) if group else os.getenv("JEV_MIN_CONFIDENCE", "0.7")
|
||||
minimum = float(os.getenv(env_key(point).replace("_PROVIDER", "_MIN_CONFIDENCE"), default_minimum))
|
||||
_number(minimum, 0, 1)
|
||||
if _diagnostics_enabled():
|
||||
logger.info("jev.decision.diagnostics point=%s status=%s confidence=%.4f minimum_confidence=%.4f questions=%s",
|
||||
point, "accepted" if confidence >= minimum else "rejected_low_confidence",
|
||||
confidence, minimum, json.dumps(diagnostic_questions, separators=(",", ":")))
|
||||
if confidence < minimum:
|
||||
# A NOUL value is P(True), not confidence in the True answer.
|
||||
# Log only numeric calibration data, never decision input or question text.
|
||||
logger.warning("decision point=%s provider=jev confidence=%.4f minimum_confidence=%.4f status=rejected predicate_probabilities=%s",
|
||||
point, confidence, minimum,
|
||||
[(name, round(probability, 4)) for name, probability in predicate_probabilities])
|
||||
raise _validation_error("CONFIDENCE_BELOW_THRESHOLD")
|
||||
result["confidence"] = confidence
|
||||
result["reason"] = "JEV typed decision (no generated explanation)."
|
||||
logger.info("decision point=%s provider=jev model=%s elapsed_ms=%.1f usage=%s gateway=%s",
|
||||
point, response.get("model"), (time.monotonic()-started)*1000,
|
||||
response.get("usage"), response.get("provider_metadata", {}).get("gateway", {}))
|
||||
return result
|
||||
|
||||
|
||||
async def invoke(llm: Any, messages: list, *, decision_point: str, decision_questions: list,
|
||||
decision_render=None, decision_generative: bool = False, decision_group: str | None = None, **kwargs: Any) -> str:
|
||||
if decision_generative or not selected(decision_point, decision_group):
|
||||
return await llm.ainvoke(messages, **kwargs)
|
||||
try:
|
||||
result = await evaluate(decision_point, messages, decision_questions, llm=llm, operation=kwargs.get("generation_name"), group=decision_group)
|
||||
return decision_render(result) if decision_render else json.dumps(result, ensure_ascii=False)
|
||||
except Exception as exc:
|
||||
# Do not log exception text, input, response bodies or credentials.
|
||||
from urllib.error import HTTPError
|
||||
from .adapters.typesafe import TypeSafeContractError
|
||||
category = (exc.code if isinstance(exc, TypeSafeContractError) else
|
||||
"HTTP_ERROR" if isinstance(exc, HTTPError) else
|
||||
"VALIDATION_ERROR" if isinstance(exc, ValueError) else
|
||||
"TRANSPORT_ERROR")
|
||||
http_status = exc.code if isinstance(exc, HTTPError) else None
|
||||
error_code = (exc.code if isinstance(exc, TypeSafeContractError) else
|
||||
str(exc) if isinstance(exc, ValueError) and
|
||||
str(exc) in {"MISSING_DECISION_ANSWERS", "DUPLICATE_OR_INVALID_ANSWERS",
|
||||
"ANSWER_TYPE_MISMATCH", "UNDECLARED_CHOICE",
|
||||
"CONFIDENCE_BELOW_THRESHOLD", "INVALID_PROBABILITY_OR_SCORE"}
|
||||
else "UNCLASSIFIED")
|
||||
http_detail = _safe_http_diagnostic(exc) if isinstance(exc, HTTPError) else "not_applicable"
|
||||
logger.warning("decision point=%s provider=jev failure=%s category=%s http_status=%s "
|
||||
"error_code=%s http_schema=%s question_types=%s",
|
||||
decision_point, type(exc).__name__, category, http_status,
|
||||
error_code, http_detail, ",".join(sorted({q.get("type", "unknown") for q in decision_questions if isinstance(q, dict)})))
|
||||
fallback = os.getenv("DECISION_FALLBACK_TO_CURRENT", "true").strip().lower()
|
||||
if fallback not in {"true", "false"}:
|
||||
raise ValueError("DECISION_FALLBACK_TO_CURRENT must be true or false")
|
||||
if fallback != "true":
|
||||
raise
|
||||
logger.info("decision point=%s provider=current fallback=true", decision_point)
|
||||
return await llm.ainvoke(messages, **kwargs)
|
||||
|
||||
|
||||
async def multi_intent(llm, messages, *, intents, text, **kwargs):
|
||||
"""Keep source-fragment extraction and request order; JEV selects each intent."""
|
||||
raw = await llm.ainvoke(messages, **kwargs)
|
||||
if not selected("multi_intent"):
|
||||
return raw
|
||||
from agent_framework.llm.structured_output import parse_json_object
|
||||
candidates = parse_json_object(raw).get("operations")
|
||||
if not isinstance(candidates, list) or not candidates:
|
||||
raise ValueError("Multi-intent extraction returned no candidates")
|
||||
if any(not isinstance(c, dict) or not str(c.get("source_text") or "").strip()
|
||||
or str(c["source_text"]) not in text for c in candidates):
|
||||
raise ValueError("Multi-intent source must be a literal user-message fragment")
|
||||
questions = [choice("operation_" + str(n), [i.name for i in intents] + ["__unsupported__"],
|
||||
"Classify ONLY this extracted objective: " + c["source_text"] + ". Use __unsupported__ for an objective outside the intent catalog.") for n, c in enumerate(candidates)]
|
||||
|
||||
class ExtractedProvider:
|
||||
usage_repository = getattr(llm, "usage_repository", None)
|
||||
telemetry = getattr(llm, "telemetry", None)
|
||||
|
||||
async def ainvoke(self, messages, **kwargs):
|
||||
return raw
|
||||
|
||||
return await invoke(ExtractedProvider(), messages, decision_point="multi_intent", decision_questions=questions,
|
||||
decision_render=lambda d: json.dumps({"confidence": d["confidence"], "operations": [
|
||||
{"intent": None if d["operation_" + str(n)] == "__unsupported__" else d["operation_" + str(n)],
|
||||
"unsupported": d["operation_" + str(n)] == "__unsupported__", "source_text": c["source_text"]}
|
||||
for n, c in enumerate(candidates)]}))
|
||||
|
||||
|
||||
async def _record(llm, point, response, operation):
|
||||
"""Account for billed responses even if validation later rejects them."""
|
||||
if llm is None:
|
||||
return
|
||||
usage = response.get("usage") or {}
|
||||
gateway = (response.get("provider_metadata") or {}).get("gateway") or {}
|
||||
model = response.get("model") or os.getenv("JEV_MODEL", "typesafe-ai/jev")
|
||||
metadata = {"provider": "jev", "model": model, "decision_point": point,
|
||||
"generation_id": gateway.get("generationId"), "gateway_cost": gateway.get("cost")}
|
||||
try:
|
||||
repository = getattr(llm, "usage_repository", None)
|
||||
if repository is not None:
|
||||
from agent_framework.billing.usage_repository import UsageRecord
|
||||
await repository.record(UsageRecord.from_usage("jev", model, operation or "decision." + point,
|
||||
{"prompt_tokens": usage.get("input_tokens", 0), "completion_tokens": usage.get("output_tokens", 0),
|
||||
"total_tokens": usage.get("total_tokens", 0), "cost_usd": gateway.get("cost", 0)}, metadata))
|
||||
telemetry = getattr(llm, "telemetry", None)
|
||||
if telemetry is not None:
|
||||
await telemetry.event("decision.jev", {**metadata, "usage": usage})
|
||||
except Exception as exc:
|
||||
logger.warning("decision accounting point=%s failure=%s", point, type(exc).__name__)
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from agent_framework.decisions import invoke as decision_invoke, choice
|
||||
|
||||
from agent_framework.localization import terms
|
||||
|
||||
from agent_framework.llm.structured_output import parse_json_object
|
||||
@@ -162,10 +164,12 @@ class GlobalSupervisorRouter:
|
||||
return fallback or self._route_by_rules(text, mode)
|
||||
prompt = self._build_supervisor_prompt(text, request)
|
||||
try:
|
||||
raw = await self.llm.ainvoke([
|
||||
raw = await decision_invoke(self.llm, [
|
||||
{"role": "system", "content": "Você é um supervisor global de backends. Responda somente JSON válido."},
|
||||
{"role": "user", "content": prompt},
|
||||
], temperature=0, profile_name="supervisor", component_name="supervisor", generation_name="llm.supervisor")
|
||||
], temperature=0, decision_point='global_supervisor',
|
||||
decision_questions=[choice("backend_id", list(self.registry.backends))],
|
||||
profile_name="supervisor", component_name="supervisor", generation_name="llm.supervisor")
|
||||
data = self._parse_json(raw)
|
||||
backend_id = str(data.get("backend") or data.get("backend_id") or "").strip()
|
||||
if backend_id not in self.registry.backends:
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from agent_framework.decisions import selected
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
@@ -152,7 +154,7 @@ class GuardrailLLMClient:
|
||||
injete o `LangfuseCallbackHandler` para que o `ChatLLM` da reescrita
|
||||
apareca como span no Langfuse.
|
||||
"""
|
||||
if self.use_mock:
|
||||
if self.use_mock and not selected("guardrail_" + task.lower()):
|
||||
return self._mock_classify(task, payload)
|
||||
|
||||
# O caminho real usa exclusivamente o provider oficial do framework.
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from agent_framework.decisions import invoke as decision_invoke, selected, predicate
|
||||
|
||||
from agent_framework.localization import terms
|
||||
|
||||
from agent_framework.llm.structured_output import parse_json_object
|
||||
@@ -369,6 +371,7 @@ async def classify_with_framework_llm(
|
||||
`profile_name`, `component_name` e `generation_name`, em vez de criar um
|
||||
cliente LLM paralelo fora da arquitetura do framework.
|
||||
"""
|
||||
decision_point = component_name or "guardrail." + task.lower()
|
||||
selected_profile = _selected_profile_for_task(task, profile_name)
|
||||
owned_llm = llm is None
|
||||
llm = _ensure_framework_llm(llm)
|
||||
@@ -379,7 +382,7 @@ async def classify_with_framework_llm(
|
||||
# provider and surfaces the bad model/provider error instead of returning a
|
||||
# local fallback result.
|
||||
force_real_from_profile = _profile_forces_real_llm(llm, selected_profile) if llm is not None else False
|
||||
if (llm is None) or (_truthy(os.getenv("USE_MOCK_LLM"), True) and not force_real_from_profile):
|
||||
if (llm is None) or (_truthy(os.getenv("USE_MOCK_LLM"), True) and not force_real_from_profile and not selected(decision_point, "guardrail_" + task.lower())):
|
||||
out = _mock_classify(task, payload)
|
||||
out.setdefault("profile_name", selected_profile)
|
||||
out.setdefault("profile_forced_real_llm", False)
|
||||
@@ -396,11 +399,16 @@ async def classify_with_framework_llm(
|
||||
else "Responda apenas JSON válido, sem markdown."
|
||||
)
|
||||
try:
|
||||
raw = await llm.ainvoke(
|
||||
raw = await decision_invoke(llm,
|
||||
[
|
||||
{"role": "system", "content": system_instruction},
|
||||
{"role": "user", "content": prompt},
|
||||
],
|
||||
decision_point=decision_point,
|
||||
decision_group="guardrail_" + task.lower(),
|
||||
decision_generative=task in {"TOXOUT", "FALLBACK"},
|
||||
decision_questions=[predicate("allowed", "Apply the supplied guardrail policy and determine whether the text is allowed. " + ("The policy digit 1 means BLOCK, 0 means ALLOW." if task == "REVPREC" else ""))],
|
||||
decision_render=lambda d: (str(int(d["allowed"]) if _BINARY_BLOCK_DIGIT.get(task, "0") == "0" else int(not d["allowed"])) if task in _BINARY_TASKS else json.dumps({**d, "label": "OK" if d["allowed"] else task})),
|
||||
profile_name=selected_profile,
|
||||
component_name=selected_component,
|
||||
generation_name=selected_generation,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user