adjustments: manual v2
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FRAMEWORK_SRC = ROOT / "libs" / "agent_framework" / "src"
|
||||
if str(FRAMEWORK_SRC) not in sys.path:
|
||||
sys.path.insert(0, str(FRAMEWORK_SRC))
|
||||
|
||||
from agent_framework.workflows.models import WorkflowDefinition
|
||||
from agent_framework.idempotency import create_idempotency_store
|
||||
|
||||
|
||||
PT = ROOT / "docs" / "developer" / "pt"
|
||||
|
||||
|
||||
def _tagged(path: Path, language: str, tag: str) -> str:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
match = re.search(rf"```{language} {re.escape(tag)}\n(.*?)```", text, re.DOTALL)
|
||||
assert match, f"bloco {tag!r} ausente em {path}"
|
||||
return match.group(1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("filename", "tag"),
|
||||
[
|
||||
("13_workflows_transacionais_avancados.md", "workflow-actions-example"),
|
||||
("13_workflows_transacionais_avancados.md", "idempotency-example"),
|
||||
("15_apresentacao_recuperacao_e_observabilidade.md", "renderer-example"),
|
||||
("16_multiagente_composicao_e_ciclo_de_vida.md", "agent-runtime-bundle-example"),
|
||||
],
|
||||
)
|
||||
def test_python_examples_compile(filename: str, tag: str) -> None:
|
||||
source = _tagged(PT / filename, "python", tag)
|
||||
ast.parse(source)
|
||||
compile(source, f"{filename}:{tag}", "exec")
|
||||
|
||||
|
||||
def test_documented_workflow_is_accepted_by_runtime_model() -> None:
|
||||
raw = yaml.safe_load(_tagged(PT / "13_workflows_transacionais_avancados.md", "yaml", "workflow-definition-example"))
|
||||
definition = WorkflowDefinition.model_validate(raw)
|
||||
assert definition.name == "devolucao_pedido"
|
||||
assert definition.version == 1
|
||||
assert [node.action for node in definition.nodes] == ["validar_pedido", "registrar_devolucao"]
|
||||
|
||||
|
||||
def test_documented_routing_and_registry_contracts() -> None:
|
||||
routing = yaml.safe_load(_tagged(PT / "14_routing_parametros_e_estado_avancados.md", "yaml", "routing-advanced-example"))
|
||||
mapping = yaml.safe_load(_tagged(PT / "14_routing_parametros_e_estado_avancados.md", "yaml", "parameter-mapping-example"))
|
||||
registry = yaml.safe_load(_tagged(PT / "16_multiagente_composicao_e_ciclo_de_vida.md", "yaml", "agents-registry-example"))
|
||||
|
||||
semantic = routing["router"]["transaction_confirmation"]["semantic_fallback"]
|
||||
assert set(semantic["allowed_values"]) == {"SIM", "NAO", "CONTINUAR"}
|
||||
assert routing["intents"][0]["mcp_tools"] == ["consultar_saldo", "consultar_movimentacoes"]
|
||||
assert mapping["mcp_parameter_mapping"]["tools"]["consultar_movimentacoes"]["extract"]["transaction_id"]["group"] == 1
|
||||
assert registry["agents"][0]["agent_id"] == registry["default_agent_id"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_documented_idempotency_example_executes_only_once() -> None:
|
||||
namespace: dict[str, object] = {}
|
||||
exec(_tagged(PT / "13_workflows_transacionais_avancados.md", "python", "idempotency-example"), namespace)
|
||||
settings = SimpleNamespace(IDEMPOTENCY_PROVIDER="memory", IDEMPOTENCY_REQUIRE_DURABLE=False)
|
||||
store = create_idempotency_store(settings, namespace="return-request")
|
||||
calls = 0
|
||||
|
||||
async def invoke():
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return {"protocol": "DEV-123"}
|
||||
|
||||
first = await namespace["execute_once"](store, "customer-1", "123", invoke)
|
||||
second = await namespace["execute_once"](store, "customer-1", "123", invoke)
|
||||
assert first == second == {"protocol": "DEV-123"}
|
||||
assert calls == 1
|
||||
|
||||
|
||||
def test_documented_renderer_masks_and_formats_output() -> None:
|
||||
namespace: dict[str, object] = {}
|
||||
exec(_tagged(PT / "15_apresentacao_recuperacao_e_observabilidade.md", "python", "renderer-example"), namespace)
|
||||
rendered = namespace["render_balance"](
|
||||
tool_name="consultar_saldo",
|
||||
result={"account_masked": "****1234", "balance": 42.5, "secret": "must-not-leak"},
|
||||
state={},
|
||||
agent_label="Financeiro",
|
||||
)
|
||||
assert rendered == "[Financeiro] Conta ****1234: saldo BRL 42.50."
|
||||
assert "must-not-leak" not in rendered
|
||||
|
||||
|
||||
def test_new_chapters_have_english_counterparts_and_are_indexed() -> None:
|
||||
pairs = {
|
||||
"13_workflows_transacionais_avancados.md": "13_advanced_transactional_workflows.md",
|
||||
"14_routing_parametros_e_estado_avancados.md": "14_advanced_routing_parameters_and_state.md",
|
||||
"15_apresentacao_recuperacao_e_observabilidade.md": "15_presentation_recovery_and_observability.md",
|
||||
"16_multiagente_composicao_e_ciclo_de_vida.md": "16_multi_agent_composition_and_lifecycle.md",
|
||||
}
|
||||
pt_index = (PT / "INDEX_DEVELOPER_GUIDE.md").read_text(encoding="utf-8")
|
||||
en_index = (ROOT / "docs" / "developer" / "en" / "INDEX_DEVELOPER_GUIDE.md").read_text(encoding="utf-8")
|
||||
for pt_name, en_name in pairs.items():
|
||||
assert (PT / pt_name).is_file() and pt_name in pt_index
|
||||
assert (ROOT / "docs" / "developer" / "en" / en_name).is_file() and en_name in en_index
|
||||
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
GUIDE = ROOT / "docs" / "EXTERNAL_GUARDRAILS_JUDGES.md"
|
||||
FRAMEWORK_SRC = ROOT / "libs" / "agent_framework" / "src"
|
||||
if str(FRAMEWORK_SRC) not in sys.path:
|
||||
sys.path.insert(0, str(FRAMEWORK_SRC))
|
||||
|
||||
from agent_framework.guardrails.pipeline import GuardrailPipeline
|
||||
from agent_framework.judges.judge import JudgePipeline
|
||||
|
||||
|
||||
def _block(tag: str) -> str:
|
||||
text = GUIDE.read_text(encoding="utf-8")
|
||||
match = re.search(rf"```(?:python|yaml) {re.escape(tag)}\n(.*?)```", text, re.DOTALL)
|
||||
assert match, f"bloco {tag!r} ausente do manual"
|
||||
return match.group(1)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def documented_agent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
|
||||
package = tmp_path / "financial_agent" / "extensions"
|
||||
package.mkdir(parents=True)
|
||||
(tmp_path / "financial_agent" / "__init__.py").write_text("", encoding="utf-8")
|
||||
(package / "__init__.py").write_text("", encoding="utf-8")
|
||||
(package / "guardrails.py").write_text(_block("external-guardrail-example"), encoding="utf-8")
|
||||
(package / "judges.py").write_text(_block("external-judge-example"), encoding="utf-8")
|
||||
(tmp_path / "guardrails.yaml").write_text(_block("external-guardrail-yaml"), encoding="utf-8")
|
||||
(tmp_path / "judges.yaml").write_text(_block("external-judge-yaml"), encoding="utf-8")
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
return tmp_path
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_documented_external_guardrail_loads_and_enforces_policy(documented_agent: Path) -> None:
|
||||
pipeline = GuardrailPipeline(config_path=str(documented_agent / "guardrails.yaml"))
|
||||
|
||||
_, allowed = await pipeline.run_tool("transfer", {"amount": 1200.0})
|
||||
_, denied = await pipeline.run_tool("transfer", {"amount": 2000.0})
|
||||
_, missing = await pipeline.run_tool("transfer", {})
|
||||
|
||||
assert allowed[0].code == "FIN_AMOUNT" and allowed[0].allowed
|
||||
assert denied[0].code == "FIN_AMOUNT" and not denied[0].allowed
|
||||
assert missing[0].code == "FIN_AMOUNT" and not missing[0].allowed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_documented_external_judge_loads_with_injection(documented_agent: Path) -> None:
|
||||
fake_llm = object()
|
||||
settings = SimpleNamespace(ENABLE_JUDGES=True)
|
||||
pipeline = JudgePipeline(
|
||||
llm=fake_llm,
|
||||
settings=settings,
|
||||
config_path=str(documented_agent / "judges.yaml"),
|
||||
)
|
||||
|
||||
passed = await pipeline.evaluate_all("Qual foi o saldo?", "O saldo foi R$ 100.", {"evidence": "R$ 100"})
|
||||
failed = await pipeline.evaluate_all("Qual foi o saldo?", "Não tenho dados.", {"evidence": "R$ 100"})
|
||||
|
||||
assert pipeline.judges[0].llm is fake_llm
|
||||
assert passed[0].name == "financial_evidence" and passed[0].passed
|
||||
assert failed[0].name == "financial_evidence" and not failed[0].passed
|
||||
Reference in New Issue
Block a user