M0 + M1 done

RI/tests/e2e/conftest.py

@@ -0,0 +1,273 @@
+"""Shared e2e fixtures: a real OPA process, a real JWKS HTTP endpoint + RS256
+token signing, and helpers to run the Gateway/Sales servers in-process
+against them. No Keycloak available in this environment (no Java) -- see
+RI/README.md for exactly what that does and does not validate.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import shutil
+import socket
+import subprocess
+import threading
+import time
+from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
+
+import jwt
+import pytest
+import uvicorn
+from cryptography.hazmat.primitives.asymmetric import rsa
+from jwt.algorithms import RSAAlgorithm
+from mcp.client.session import ClientSession
+from mcp.client.streamable_http import (  # type: ignore[attr-defined]
+    create_mcp_http_client,
+    streamable_http_client,
+)
+from mcp.shared.exceptions import MCPError
+
+REPO_ROOT = Path(__file__).parents[2]
+POLICIES_DIR = REPO_ROOT / "config" / "policies"
+
+ISSUER = "https://keycloak.example/realms/emcp"
+AUDIENCE = "emcp-gateway"
+KID = "test-key-1"
+
+GATEWAY_PORT = 18000
+SALES_PORT = 18100
+
+
+def free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return int(s.getsockname()[1])
+
+
+# --- Real OPA process (EP-03) --------------------------------------------
+
+
+@pytest.fixture(scope="session")
+def opa_url() -> Iterator[str]:
+    opa_binary = shutil.which(os.environ.get("OPA_BINARY", "opa"))
+    if opa_binary is None:
+        pytest.skip(
+            "opa binary not found on PATH (set OPA_BINARY or install it -- see RI/README.md)"
+        )
+
+    port = free_port()
+    process = subprocess.Popen(
+        [opa_binary, "run", "--server", "--addr", f"127.0.0.1:{port}", str(POLICIES_DIR)],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    base_url = f"http://127.0.0.1:{port}"
+    try:
+        import httpx
+
+        for _ in range(100):
+            try:
+                httpx.get(f"{base_url}/health", timeout=1)
+                break
+            except httpx.HTTPError:
+                time.sleep(0.05)
+        else:
+            raise RuntimeError("opa server did not become healthy in time")
+        yield base_url
+    finally:
+        process.terminate()
+        process.wait(timeout=5)
+
+
+# --- Real JWKS HTTP endpoint + RS256 signing (EP-01-T03) ------------------
+
+
+class _JWKSHandler(BaseHTTPRequestHandler):
+    jwks_document: dict[str, object] = {}
+
+    def do_GET(self) -> None:  # noqa: N802
+        body = json.dumps(self.jwks_document).encode("utf-8")
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, format: str, *args: object) -> None:  # noqa: A002
+        pass
+
+
+@pytest.fixture(scope="session")
+def rsa_keypair() -> tuple[rsa.RSAPrivateKey, rsa.RSAPublicKey]:
+    private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    return private_key, private_key.public_key()
+
+
+@pytest.fixture(scope="session")
+def jwks_url(rsa_keypair: tuple[rsa.RSAPrivateKey, rsa.RSAPublicKey]) -> Iterator[str]:
+    _, public_key = rsa_keypair
+    jwk = RSAAlgorithm.to_jwk(public_key, as_dict=True)
+    jwk.update({"kid": KID, "use": "sig", "alg": "RS256"})
+    _JWKSHandler.jwks_document = {"keys": [jwk]}
+
+    server = HTTPServer(("127.0.0.1", 0), _JWKSHandler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    try:
+        yield f"http://127.0.0.1:{server.server_port}/jwks"
+    finally:
+        server.shutdown()
+        thread.join(timeout=5)
+
+
+@pytest.fixture
+def sign_token(
+    rsa_keypair: tuple[rsa.RSAPrivateKey, rsa.RSAPublicKey],
+) -> Callable[..., str]:
+    private_key, _ = rsa_keypair
+
+    def _sign(
+        client_id: str,
+        *,
+        issuer: str = ISSUER,
+        audience: str = AUDIENCE,
+        expires_in_seconds: int = 300,
+        signing_key: rsa.RSAPrivateKey | None = None,
+    ) -> str:
+        now = int(time.time())
+        claims = {
+            "iss": issuer,
+            "aud": audience,
+            "sub": f"service-account-{client_id}",
+            "azp": client_id,
+            "iat": now,
+            "exp": now + expires_in_seconds,
+        }
+        return jwt.encode(
+            claims, signing_key or private_key, algorithm="RS256", headers={"kid": KID}
+        )
+
+    return _sign
+
+
+# --- Running Gateway + Sales servers, wired to the fixtures above --------
+
+
+class RunningServer:
+    def __init__(self, config: uvicorn.Config) -> None:
+        self.server = uvicorn.Server(config)
+        self._task: asyncio.Task[None] | None = None
+
+    async def start(self) -> None:
+        self._task = asyncio.create_task(self.server.serve())
+        while not self.server.started:
+            await asyncio.sleep(0.01)
+        # See test_walking_skeleton.py's original note: give the ASGI app's
+        # own lifespan (session manager task group) one extra tick.
+        await asyncio.sleep(0.05)
+
+    async def stop(self) -> None:
+        self.server.should_exit = True
+        assert self._task is not None
+        await self._task
+
+
+@pytest.fixture
+async def governed_stack(
+    monkeypatch: pytest.MonkeyPatch, opa_url: str, jwks_url: str
+) -> AsyncIterator[None]:
+    """Real OPA + real JWKS-backed OIDC auth, wired into fresh Gateway/Sales
+    instances for one test (EP-01/EP-03/EP-04/EP-05 all real, only Keycloak
+    itself is substituted by the JWKS server -- see RI/README.md)."""
+    monkeypatch.setenv("SALES_DOMAIN_URL", f"http://127.0.0.1:{SALES_PORT}/mcp")
+    monkeypatch.setenv("OPA_URL", opa_url)
+    monkeypatch.setenv("OIDC_JWKS_URL", jwks_url)
+    monkeypatch.setenv("OIDC_ISSUER", ISSUER)
+    monkeypatch.setenv("OIDC_AUDIENCE", AUDIENCE)
+
+    import example_mcp_servers.sales_domain.server as sales_module
+
+    sales_config = uvicorn.Config(
+        sales_module.server.streamable_http_app(host="127.0.0.1"),
+        host="127.0.0.1",
+        port=SALES_PORT,
+        log_level="warning",
+    )
+    sales_server = RunningServer(sales_config)
+    await sales_server.start()
+
+    import emcp_bus.gateway.server as gateway_module
+
+    auth_settings, token_verifier = gateway_module.build_auth()
+    gateway_config = uvicorn.Config(
+        gateway_module.server.streamable_http_app(
+            host="127.0.0.1", auth=auth_settings, token_verifier=token_verifier
+        ),
+        host="127.0.0.1",
+        port=GATEWAY_PORT,
+        log_level="warning",
+    )
+    gateway_server = RunningServer(gateway_config)
+    await gateway_server.start()
+
+    try:
+        yield
+    finally:
+        await gateway_server.stop()
+        await sales_server.stop()
+
+
+async def run_against_gateway[ResultT](
+    operation: Callable[[ClientSession], Awaitable[ResultT]],
+    *,
+    token: str | None = None,
+) -> ResultT:
+    """Connect to the gateway (optionally authenticated) and run `operation(session)`.
+
+    Retries on `MCPError` ("SSE stream ended without a response" has been
+    observed, rarely, under this in-process dual-uvicorn test setup under
+    scheduler contention) -- a test-harness robustness concern, not a
+    Gateway defect: production deployments run one process per service.
+    """
+    headers = {"Authorization": f"Bearer {token}"} if token else None
+    url = f"http://127.0.0.1:{GATEWAY_PORT}/mcp"
+    last_error: MCPError | None = None
+    for attempt in range(3):
+        if attempt:
+            await asyncio.sleep(0.1 * attempt)
+        try:
+            async with (
+                create_mcp_http_client(headers=headers) as http_client,
+                streamable_http_client(url, http_client=http_client) as (read, write),
+                ClientSession(read, write) as session,
+            ):
+                await session.initialize()
+                return await operation(session)
+        except MCPError as exc:
+            last_error = exc
+        except BaseExceptionGroup as group:
+            # Each nested anyio TaskGroup (`streamable_http_client`'s, then
+            # `ClientSession`'s) wraps the one below it in its own
+            # ExceptionGroup, so a handler-raised MCPError (e.g. an auth
+            # rejection during `initialize`) can arrive doubly-wrapped.
+            # Unwrap fully so callers can `pytest.raises(MCPError)`
+            # uniformly, same as for one raised directly.
+            unwrapped = _unwrap_single_mcp_error(group)
+            if unwrapped is not None:
+                last_error = unwrapped
+            else:
+                raise
+    assert last_error is not None
+    raise last_error
+
+
+def _unwrap_single_mcp_error(exc: BaseException) -> MCPError | None:
+    """If `exc` is an MCPError, or a (possibly nested) ExceptionGroup wrapping
+    exactly one, return it; otherwise `None`."""
+    if isinstance(exc, MCPError):
+        return exc
+    if isinstance(exc, BaseExceptionGroup) and len(exc.exceptions) == 1:
+        return _unwrap_single_mcp_error(exc.exceptions[0])
+    return None

RI/tests/e2e/test_governed_gateway.py

@@ -0,0 +1,242 @@
+"""EP-05-T02/T04 governed Gateway: the actual security invariants of
+README.md §45 ("Critérios de aceite de segurança"), run against the real
+chain -- real OIDC token validation (JWKS server standing in for Keycloak),
+real OPA-backed PDP, real EntitlementManager loading the real config/
+YAML files.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import pytest
+from mcp.shared.exceptions import MCPError
+
+from tests.e2e.conftest import run_against_gateway
+
+SALES_READ_CLIENT_ID = "sales-read-agent"
+SALES_WRITE_CLIENT_ID = "sales-write-agent"
+FINANCE_PAYMENTS_CLIENT_ID = "finance-payments-agent"
+
+
+async def test_teste1_unauthorized_discovery(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    """README.md §45 Teste 1: finance.createPayment MUST NOT be returned to Sales.Read."""
+    token = sign_token(SALES_READ_CLIENT_ID)
+
+    result = await run_against_gateway(lambda session: session.list_tools(), token=token)
+
+    tool_names = {tool.name for tool in result.tools}
+    assert "finance.payment.execute" not in tool_names
+    assert "sales.quote.create" not in tool_names  # Sales.Write-only, not Sales.Read
+
+
+async def test_teste2_direct_unauthorized_call(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    """README.md §45 Teste 2: calling a known-but-unentitled tool directly, without
+    ever having seen it in tools/list, must still DENY."""
+    token = sign_token(SALES_READ_CLIENT_ID)
+
+    result = await run_against_gateway(
+        lambda session: session.call_tool("finance.payment.execute", {"amount": 1_000_000}),
+        token=token,
+    )
+
+    assert result.is_error is True
+    assert "Access denied" in _text(result)
+
+
+async def test_teste4_agent_role_spoofing_has_no_effect(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    """README.md §45 Teste 4: nothing the caller declares in tool arguments
+    (an agent-shaped 'role' claim in this case) can expand entitlement."""
+    token = sign_token(SALES_READ_CLIENT_ID)
+
+    result = await run_against_gateway(
+        lambda session: session.call_tool(
+            "finance.payment.execute",
+            {"amount": 100, "role": "finance-admin", "client_profile": "Finance.Payments"},
+        ),
+        token=token,
+    )
+
+    assert result.is_error is True
+
+
+async def test_unauthenticated_request_sees_no_tools_and_is_denied(
+    governed_stack: None,
+) -> None:
+    """README.md §5: 'No entitlement -> no discovery -> no execution'.
+
+    With OIDC configured, the SDK's own `RequireAuthMiddleware` rejects a
+    connection with no bearer token before the MCP session handshake
+    (`initialize`) ever completes -- stricter than this Gateway's own
+    handler-level `access_token is None` checks (`on_list_tools`/
+    `on_call_tool`), which exist as the deny-by-default backstop for the
+    *other* supported deployment mode: no OIDC configured at all (see
+    `GatewayConfig`/`build_auth` in emcp_bus/gateway/server.py), where every
+    request reaches the handlers unauthenticated and those checks are what
+    enforces "no discovery, no execution" instead.
+    """
+    with pytest.raises(MCPError):
+        await run_against_gateway(lambda session: session.list_tools())
+
+
+async def test_expired_token_is_denied(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    """README.md §45 Teste 3 (adjacent)/§46: expired token, adversarial scenario."""
+    token = sign_token(SALES_READ_CLIENT_ID, expires_in_seconds=-3600)
+
+    with pytest.raises(MCPError):
+        await run_against_gateway(lambda session: session.list_tools(), token=token)
+
+
+async def test_wrong_audience_token_is_denied(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    """README.md §45 Teste 8: a token issued for a different resource server."""
+    token = sign_token(SALES_READ_CLIENT_ID, audience="some-other-service")
+
+    with pytest.raises(MCPError):
+        await run_against_gateway(lambda session: session.list_tools(), token=token)
+
+
+async def test_forged_signature_token_is_denied(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    """README.md §46: stolen/forged token, signed by a key not in the JWKS."""
+    from cryptography.hazmat.primitives.asymmetric import rsa
+
+    forged_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    token = sign_token(SALES_READ_CLIENT_ID, signing_key=forged_key)
+
+    with pytest.raises(MCPError):
+        await run_against_gateway(lambda session: session.list_tools(), token=token)
+
+
+async def test_transaction_policy_denies_over_limit_even_though_entitled(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    """README.md §25 Passo 2: sales.quote.create IS in Sales.Write's entitlement,
+    but the transaction policy still denies an amount above the profile's limit."""
+    token = sign_token(SALES_WRITE_CLIENT_ID)
+
+    result = await run_against_gateway(
+        lambda session: session.call_tool(
+            "sales.quote.create", {"amount": 999_999_999, "region": "BR-SP"}
+        ),
+        token=token,
+    )
+
+    assert result.is_error is True
+    assert "TRANSACTION_POLICY_DENIED" in _text(result)
+
+
+async def test_require_approval_is_denied_pending_ep14(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    """README.md §26: Finance.Payments allows payment.execute below the approval
+    threshold (README.md §7.2/§17 profile) but must never silently upgrade a
+    REQUIRE_APPROVAL to ALLOW -- and EP-14 (approval workflow) does not exist
+    yet in this milestone, so the only safe outcome is DENY."""
+    token = sign_token(FINANCE_PAYMENTS_CLIENT_ID)
+
+    result = await run_against_gateway(
+        lambda session: session.call_tool("finance.payment.execute", {"amount": 50_000}),
+        token=token,
+    )
+
+    assert result.is_error is True
+    assert "REQUIRE_APPROVAL" in _text(result)
+
+
+async def test_gateway_lifespan_fails_closed_when_pdp_unreachable(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """README.md §38: PDP indisponível -> fail closed, never ALLOW-by-omission.
+
+    A Gateway that cannot sync entitlements to its PDP at startup must not
+    come up looking healthy (EP-05-T06): its `lifespan` must raise, not swallow
+    the error and yield a state built on an empty/stale entitlement sync.
+    """
+    from tests.e2e.conftest import free_port
+
+    monkeypatch.setenv("OPA_URL", f"http://127.0.0.1:{free_port()}")  # nothing listening here
+
+    import emcp_bus.gateway.server as gateway_module
+    from emcp_bus.pdp.client import PDPUnavailableError
+
+    with pytest.raises(PDPUnavailableError):
+        async with gateway_module.lifespan(gateway_module.server):
+            pytest.fail("lifespan must not yield a state when the PDP sync failed")
+
+
+async def test_deny_by_default_when_no_oidc_provider_is_configured(
+    monkeypatch: pytest.MonkeyPatch, opa_url: str
+) -> None:
+    """README.md §5: the *other* supported deployment mode (no OIDC provider
+    configured at all -- `OIDC_JWKS_URL`/`OIDC_ISSUER` unset). No
+    `RequireAuthMiddleware` is installed in this mode (see `build_auth`), so
+    every request reaches the handlers with `get_access_token() is None`,
+    and it is the Gateway's own handler-level checks that must deny by
+    default -- this is the code path Teste 1-style discovery filtering and
+    Teste 2-style execution denial fall back to without an IdP.
+    """
+    import uvicorn
+
+    from tests.e2e.conftest import GATEWAY_PORT, SALES_PORT, RunningServer, run_against_gateway
+
+    monkeypatch.setenv("SALES_DOMAIN_URL", f"http://127.0.0.1:{SALES_PORT}/mcp")
+    monkeypatch.setenv("OPA_URL", opa_url)
+    monkeypatch.delenv("OIDC_JWKS_URL", raising=False)
+    monkeypatch.delenv("OIDC_ISSUER", raising=False)
+
+    import example_mcp_servers.sales_domain.server as sales_module
+
+    sales_server = RunningServer(
+        uvicorn.Config(
+            sales_module.server.streamable_http_app(host="127.0.0.1"),
+            host="127.0.0.1",
+            port=SALES_PORT,
+            log_level="warning",
+        )
+    )
+    await sales_server.start()
+
+    import emcp_bus.gateway.server as gateway_module
+
+    auth_settings, token_verifier = gateway_module.build_auth()
+    assert (auth_settings, token_verifier) == (None, None)  # confirms the mode under test
+
+    gateway_server = RunningServer(
+        uvicorn.Config(
+            gateway_module.server.streamable_http_app(
+                host="127.0.0.1", auth=auth_settings, token_verifier=token_verifier
+            ),
+            host="127.0.0.1",
+            port=GATEWAY_PORT,
+            log_level="warning",
+        )
+    )
+    await gateway_server.start()
+    try:
+        list_result = await run_against_gateway(lambda session: session.list_tools())
+        assert list_result.tools == []
+
+        call_result = await run_against_gateway(
+            lambda session: session.call_tool("sales.customer.get", {"customer_id": "cust-001"})
+        )
+        assert call_result.is_error is True
+        assert "UNAUTHENTICATED" in _text(call_result)
+    finally:
+        await gateway_server.stop()
+        await sales_server.stop()
+
+
+def _text(result: object) -> str:
+    content = getattr(result, "content", [])
+    return " ".join(getattr(item, "text", "") for item in content)

RI/tests/e2e/test_walking_skeleton.py

@@ -0,0 +1,105 @@
+"""M0 walking skeleton, now exercised through M1's real enforcement chain
+(docs/RI-PLANNING.md, Marco M0 + M1: EP-01/EP-03/EP-04/EP-05).
+
+Client -> Gateway (real OIDC auth + real OPA-backed entitlement) -> (static
+routing) -> example Sales MCP server -> response, over real HTTP
+(streamable-http), with a single OTel trace_id correlating spans across both
+processes. Authenticates as `sales-read-agent` (Sales.Read, README.md §7.1)
+calling only tools within its own entitlement -- the denial paths (no token,
+wrong tool, expired/forged token, PDP down) are tests/e2e/test_governed_gateway.py.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from opentelemetry.sdk.trace import ReadableSpan
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import SpanKind
+
+from tests.e2e.conftest import run_against_gateway
+
+SALES_READ_CLIENT_ID = "sales-read-agent"
+
+
+async def test_tools_list_is_filtered_to_sales_read_entitlement(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    token = sign_token(SALES_READ_CLIENT_ID)
+
+    result = await run_against_gateway(lambda session: session.list_tools(), token=token)
+
+    # README.md §7.1: Sales.Read sees exactly these four -- not
+    # sales.quote.create/update (Sales.Write-only) and not any Finance tool.
+    assert {tool.name for tool in result.tools} == {
+        "sales.customer.search",
+        "sales.customer.get",
+        "sales.order.get",
+        "sales.inventory.check",
+    }
+
+
+async def test_tools_call_reaches_the_backend_and_returns_its_result(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    token = sign_token(SALES_READ_CLIENT_ID)
+
+    result = await run_against_gateway(
+        lambda session: session.call_tool("sales.customer.get", {"customer_id": "cust-001"}),
+        token=token,
+    )
+
+    assert result.is_error is not True
+    assert result.structured_content == {
+        "customer_id": "cust-001",
+        "name": "Aurora Ltda",
+        "region": "BR-SP",
+    }
+
+
+async def test_a_single_trace_id_correlates_gateway_and_backend_spans(
+    governed_stack: None, sign_token: Callable[..., str]
+) -> None:
+    """Both hops of one `tools/call` must land on the same trace_id.
+
+    This test runs both server processes in-process (in-process uvicorn, for
+    speed), so they share one OTel global TracerProvider/resource -- unlike a
+    real multi-process deployment (verified manually: each process gets its
+    own `service.name` resource), so `resource.attributes["service.name"]`
+    cannot be used here to tell the two hops apart. `SpanKind.SERVER` can:
+    the gateway receiving the call from the client, and the Sales server
+    receiving it from the gateway, are each a distinct SERVER-kind
+    `tools/call ...` span -- exactly two (or more, under a rare transport
+    retry -- see `run_against_gateway`) of them sharing one trace_id proves
+    the request was correlated across both hops.
+    """
+    from opentelemetry import trace
+
+    exporter = InMemorySpanExporter()
+    trace.get_tracer_provider().add_span_processor(SimpleSpanProcessor(exporter))  # type: ignore[attr-defined]
+
+    token = sign_token(SALES_READ_CLIENT_ID)
+    await run_against_gateway(
+        lambda session: session.call_tool("sales.customer.get", {"customer_id": "cust-001"}),
+        token=token,
+    )
+
+    spans: tuple[ReadableSpan, ...] = exporter.get_finished_spans()
+    server_tool_call_spans = [
+        span
+        for span in spans
+        if span.name.startswith("tools/call") and span.kind == SpanKind.SERVER
+    ]
+    trace_ids = {span.context.trace_id for span in server_tool_call_spans}
+
+    assert len(trace_ids) == 1, (
+        "Expected exactly one trace_id shared by both hops, got "
+        f"{[f'{t:#x}' for t in trace_ids]} across spans {[s.name for s in server_tool_call_spans]}"
+    )
+    assert len(server_tool_call_spans) >= 2, (
+        "Expected at least one SERVER 'tools/call' span for the gateway hop and one "
+        f"for the backend hop, got {len(server_tool_call_spans)}: "
+        f"{[s.name for s in server_tool_call_spans]} -- see Marco M0 acceptance "
+        "criterion in docs/RI-PLANNING.md"
+    )