new feature: External guardrails/judges
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
FW = ROOT / "agent_framework_oci" / "libs" / "agent_framework" / "src"
|
||||
if str(FW) not in sys.path:
|
||||
sys.path.insert(0, str(FW))
|
||||
|
||||
from agent_framework.llm.providers import MockLLMProvider
|
||||
from agent_framework.observability.code_mapper import ObservabilityCodeMapper
|
||||
|
||||
|
||||
class _CaptureGeneration:
|
||||
def set_output(self, value): pass
|
||||
def set_usage(self, value): pass
|
||||
def set_metadata(self, **value): pass
|
||||
|
||||
|
||||
class _TelemetryWithoutOwnNormalization:
|
||||
"""Capture the name exactly as received from the LLM provider."""
|
||||
def __init__(self):
|
||||
self.code_mapper = ObservabilityCodeMapper({
|
||||
"guardrail.dlex_in": "GRL.004",
|
||||
"guardrail.tox": "GRL.005",
|
||||
})
|
||||
self.names = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def generation_span(self, **attrs):
|
||||
self.names.append(attrs["name"])
|
||||
yield _CaptureGeneration()
|
||||
|
||||
|
||||
async def test_provider_maps_guardrail_name_before_telemetry():
|
||||
telemetry = _TelemetryWithoutOwnNormalization()
|
||||
provider = MockLLMProvider(telemetry=telemetry)
|
||||
await provider.ainvoke(
|
||||
[{"role": "user", "content": "x"}],
|
||||
generation_name="guardrail.dlex_in",
|
||||
component_name="guardrail.dlex_in",
|
||||
)
|
||||
assert telemetry.names == ["GRL.004"]
|
||||
|
||||
|
||||
def test_mapping_path_can_resolve_from_python_import_root(tmp_path, monkeypatch):
|
||||
project = tmp_path / "agent"
|
||||
config = project / "config"
|
||||
config.mkdir(parents=True)
|
||||
mapping = config / "observability_mapping.yaml"
|
||||
mapping.write_text("mappings:\n guardrail.dlex_in: GRL.004\n", encoding="utf-8")
|
||||
elsewhere = tmp_path / "runner"
|
||||
elsewhere.mkdir()
|
||||
monkeypatch.chdir(elsewhere)
|
||||
monkeypatch.setattr(sys, "path", [str(project), *sys.path])
|
||||
mapper = ObservabilityCodeMapper.from_yaml("./config/observability_mapping.yaml")
|
||||
assert mapper.map("guardrail.dlex_in") == "GRL.004"
|
||||
Reference in New Issue
Block a user