setup environment

infra/oci/nat.py

@@ -0,0 +1,93 @@
+"""Liga e desliga a saída temporária por NAT Gateway (tarefas 9, 17 e 78; decisão Q3).
+
+Uso:
+  PYTHONPATH=infra/oci .venv/bin/python infra/oci/nat.py on            # cria/desbloqueia o NAT e abre a rota
+  PYTHONPATH=infra/oci .venv/bin/python infra/oci/nat.py off           # fecha a rota e bloqueia o NAT
+  PYTHONPATH=infra/oci .venv/bin/python infra/oci/nat.py off --delete  # fecha a rota e apaga o NAT (tarefa 78)
+
+O NAT só deve ficar ativo durante a aquisição de dependências e pesos.
+Ao final, regrava .secrets/artefatos-OCI.md.
+"""
+import sys
+
+import oci
+
+import provision
+from session import load_config
+
+NAT_NAME = "dmb-nat-temporario"
+ANYWHERE = "0.0.0.0/0"
+
+
+def lookup(vn, cid):
+    vcn = next(v for v in vn.list_vcns(cid, display_name="dmb-vcn").data if v.lifecycle_state == "AVAILABLE")
+    rt = next(r for r in vn.list_route_tables(cid, vcn_id=vcn.id, display_name="dmb-rt-privada").data
+              if r.lifecycle_state == "AVAILABLE")
+    sl = next(s for s in vn.list_security_lists(cid, vcn_id=vcn.id, display_name="dmb-sl-privada").data
+              if s.lifecycle_state == "AVAILABLE")
+    nat = next((n for n in vn.list_nat_gateways(cid, vcn_id=vcn.id, display_name=NAT_NAME).data
+                if n.lifecycle_state == "AVAILABLE"), None)
+    return vcn, rt, sl, nat
+
+
+def nat_on(vn, cid):
+    vcn, rt, sl, nat = lookup(vn, cid)
+    if nat is None:
+        nat = vn.create_nat_gateway(oci.core.models.CreateNatGatewayDetails(
+            compartment_id=cid, vcn_id=vcn.id, display_name=NAT_NAME,
+            freeform_tags=provision.TAGS)).data
+        nat = oci.wait_until(vn, vn.get_nat_gateway(nat.id), "lifecycle_state", "AVAILABLE").data
+    elif nat.block_traffic:
+        vn.update_nat_gateway(nat.id, oci.core.models.UpdateNatGatewayDetails(block_traffic=False))
+
+    if not any(r.destination == ANYWHERE for r in rt.route_rules):
+        vn.update_route_table(rt.id, oci.core.models.UpdateRouteTableDetails(route_rules=rt.route_rules + [
+            oci.core.models.RouteRule(destination=ANYWHERE, destination_type="CIDR_BLOCK",
+                                      network_entity_id=nat.id, description="TEMPORÁRIO: aquisição de artefatos")]))
+    if not any(r.destination == ANYWHERE for r in sl.egress_security_rules):
+        vn.update_security_list(sl.id, oci.core.models.UpdateSecurityListDetails(
+            ingress_security_rules=sl.ingress_security_rules,
+            egress_security_rules=sl.egress_security_rules + [oci.core.models.EgressSecurityRule(
+                destination=ANYWHERE, protocol="6", description="TEMPORÁRIO: HTTPS para aquisição",
+                tcp_options=oci.core.models.TcpOptions(
+                    destination_port_range=oci.core.models.PortRange(min=443, max=443)))]))
+    print(f"NAT ativo: {nat.id}")
+
+
+def nat_off(vn, cid, delete):
+    _, rt, sl, nat = lookup(vn, cid)
+    rules = [r for r in rt.route_rules if r.destination != ANYWHERE]
+    if len(rules) != len(rt.route_rules):
+        vn.update_route_table(rt.id, oci.core.models.UpdateRouteTableDetails(route_rules=rules))
+    egress = [r for r in sl.egress_security_rules if r.destination != ANYWHERE]
+    if len(egress) != len(sl.egress_security_rules):
+        vn.update_security_list(sl.id, oci.core.models.UpdateSecurityListDetails(
+            ingress_security_rules=sl.ingress_security_rules, egress_security_rules=egress))
+    if nat is None:
+        print("NAT inexistente; rota e egresso fechados")
+        return
+    if delete:
+        vn.delete_nat_gateway(nat.id)
+        print(f"NAT apagado: {nat.id}")
+    else:
+        vn.update_nat_gateway(nat.id, oci.core.models.UpdateNatGatewayDetails(block_traffic=True))
+        print(f"NAT bloqueado: {nat.id}")
+
+
+def main():
+    if len(sys.argv) < 2 or sys.argv[1] not in ("on", "off"):
+        sys.exit(__doc__)
+    cfg = load_config()
+    home = load_config(region="us-ashburn-1")
+    cid = provision.find_compartment(oci.identity.IdentityClient(home), home["tenancy"])
+    vn = oci.core.VirtualNetworkClient(cfg)
+    if sys.argv[1] == "on":
+        nat_on(vn, cid)
+    else:
+        nat_off(vn, cid, delete="--delete" in sys.argv)
+    provision.artifacts.clear()
+    provision.main()
+
+
+if __name__ == "__main__":
+    main()

infra/oci/provision.py

@@ -0,0 +1,231 @@
+"""Provisionamento idempotente da fase F0 (tarefas 1, 2, 5, 6, 7, 8).
+
+Uso: PYTHONPATH=infra/oci .venv/bin/python infra/oci/provision.py
+Registra os artefatos criados em .secrets/artefatos-OCI.md.
+"""
+import time
+from datetime import datetime, timezone
+
+import oci
+
+from session import CRED_DIR, REGION, load_config
+
+PARENT_PATH = "root/<compartimento-pai>"
+COMPARTMENT_NAME = "decision-models"
+TAGS = {"projeto": "benchmark-decisoes-ptbr", "ambiente": "benchmark"}  # centro_custo pendente (Q13)
+BUCKETS = ["entrada", "artefatos", "resultados", "logs-imutaveis"]
+VCN_CIDR = "10.20.0.0/16"
+SUBNET_CIDR = "10.20.1.0/24"
+ARTIFACTS_FILE = CRED_DIR.parent / "artefatos-OCI.md"
+
+artifacts = []
+
+
+def record(kind, name, ocid, region):
+    artifacts.append((kind, name, ocid, region))
+
+
+def find_compartment(idn, tenancy):
+    allc = oci.pagination.list_call_get_all_results(
+        idn.list_compartments, tenancy, compartment_id_in_subtree=True,
+        access_level="ACCESSIBLE", lifecycle_state="ACTIVE").data
+    byid = {c.id: c for c in allc}
+
+    def path(c):
+        parts = []
+        while c:
+            parts.append(c.name)
+            c = byid.get(c.compartment_id)
+        return "root/" + "/".join(reversed(parts))
+
+    parent = next(c for c in allc if path(c) == PARENT_PATH)
+    comp = next((c for c in allc if c.compartment_id == parent.id and c.name == COMPARTMENT_NAME), None)
+    if comp is None:
+        comp = idn.create_compartment(oci.identity.models.CreateCompartmentDetails(
+            compartment_id=parent.id, name=COMPARTMENT_NAME, freeform_tags=TAGS,
+            description="Benchmark de modelos de decisão PT-BR para roteamento em orquestrador multiagente")).data
+        for _ in range(30):
+            try:
+                if idn.get_compartment(comp.id).data.lifecycle_state == "ACTIVE":
+                    break
+            except oci.exceptions.ServiceError as e:
+                if e.status != 404:
+                    raise
+            time.sleep(10)
+    record("Compartment", f"{PARENT_PATH}/{COMPARTMENT_NAME}", comp.id, "global (home us-ashburn-1)")
+    return comp.id
+
+
+def ensure_ds_project(cfg, cid):
+    ds = oci.data_science.DataScienceClient(cfg)
+    name = "benchmark-decisoes-ptbr"
+    proj = next((p for p in ds.list_projects(cid, lifecycle_state="ACTIVE").data if p.display_name == name), None)
+    if proj is None:
+        proj = ds.create_project(oci.data_science.models.CreateProjectDetails(
+            compartment_id=cid, display_name=name, freeform_tags=TAGS,
+            description="Benchmark de modelos de decisão PT-BR")).data
+    record("Data Science Project", name, proj.id, REGION)
+
+
+def ensure_buckets(cfg, cid):
+    os_ = oci.object_storage.ObjectStorageClient(cfg)
+    ns = os_.get_namespace().data
+    existing = {b.name for b in oci.pagination.list_call_get_all_results(os_.list_buckets, ns, cid).data}
+    for short in BUCKETS:
+        name = f"dmb-{short}"
+        if name not in existing:
+            os_.create_bucket(ns, oci.object_storage.models.CreateBucketDetails(
+                name=name, compartment_id=cid, public_access_type="NoPublicAccess",
+                storage_tier="Standard", versioning="Enabled", freeform_tags=TAGS))
+        b = os_.get_bucket(ns, name).data
+        record("Object Storage Bucket", f"{name} (namespace {ns})", b.id, REGION)
+
+
+def ensure_network(cfg, cid):
+    vn = oci.core.VirtualNetworkClient(cfg)
+    comp = oci.wait_until  # alias curto
+
+    vcn = next((v for v in vn.list_vcns(cid, display_name="dmb-vcn").data
+                if v.lifecycle_state == "AVAILABLE"), None)
+    if vcn is None:
+        vcn = vn.create_vcn(oci.core.models.CreateVcnDetails(
+            compartment_id=cid, display_name="dmb-vcn", cidr_blocks=[VCN_CIDR],
+            dns_label="dmbvcn", freeform_tags=TAGS)).data
+        vcn = comp(vn, vn.get_vcn(vcn.id), "lifecycle_state", "AVAILABLE").data
+    record("VCN", f"dmb-vcn ({VCN_CIDR})", vcn.id, REGION)
+
+    svc = next(s for s in vn.list_services().data if s.cidr_block.startswith("all-") and "services-in-oracle-services-network" in s.cidr_block)
+    sgw = next((g for g in vn.list_service_gateways(cid, vcn_id=vcn.id).data
+                if g.lifecycle_state == "AVAILABLE"), None)
+    if sgw is None:
+        sgw = vn.create_service_gateway(oci.core.models.CreateServiceGatewayDetails(
+            compartment_id=cid, vcn_id=vcn.id, display_name="dmb-sgw", freeform_tags=TAGS,
+            services=[oci.core.models.ServiceIdRequestDetails(service_id=svc.id)])).data
+        sgw = comp(vn, vn.get_service_gateway(sgw.id), "lifecycle_state", "AVAILABLE").data
+    record("Service Gateway", f"dmb-sgw ({svc.cidr_block})", sgw.id, REGION)
+
+    rt = next((r for r in vn.list_route_tables(cid, vcn_id=vcn.id, display_name="dmb-rt-privada").data
+               if r.lifecycle_state == "AVAILABLE"), None)
+    if rt is None:
+        rt = vn.create_route_table(oci.core.models.CreateRouteTableDetails(
+            compartment_id=cid, vcn_id=vcn.id, display_name="dmb-rt-privada", freeform_tags=TAGS,
+            route_rules=[oci.core.models.RouteRule(
+                destination=svc.cidr_block, destination_type="SERVICE_CIDR_BLOCK",
+                network_entity_id=sgw.id, description="Object Storage e serviços OCI via Service Gateway")])).data
+        rt = comp(vn, vn.get_route_table(rt.id), "lifecycle_state", "AVAILABLE").data
+    record("Route Table", "dmb-rt-privada (somente Service Gateway)", rt.id, REGION)
+
+    sl = next((s for s in vn.list_security_lists(cid, vcn_id=vcn.id, display_name="dmb-sl-privada").data
+               if s.lifecycle_state == "AVAILABLE"), None)
+    if sl is None:
+        sl = vn.create_security_list(oci.core.models.CreateSecurityListDetails(
+            compartment_id=cid, vcn_id=vcn.id, display_name="dmb-sl-privada", freeform_tags=TAGS,
+            egress_security_rules=[
+                oci.core.models.EgressSecurityRule(
+                    destination=svc.cidr_block, destination_type="SERVICE_CIDR_BLOCK", protocol="6",
+                    tcp_options=oci.core.models.TcpOptions(destination_port_range=oci.core.models.PortRange(min=443, max=443)),
+                    description="HTTPS para serviços OCI"),
+                oci.core.models.EgressSecurityRule(destination=VCN_CIDR, protocol="all", description="Tráfego interno da VCN"),
+            ],
+            ingress_security_rules=[
+                oci.core.models.IngressSecurityRule(source=VCN_CIDR, protocol="all", description="Tráfego interno da VCN"),
+            ])).data
+        sl = comp(vn, vn.get_security_list(sl.id), "lifecycle_state", "AVAILABLE").data
+    record("Security List", "dmb-sl-privada (egresso só para serviços OCI e VCN)", sl.id, REGION)
+
+    sub = next((s for s in vn.list_subnets(cid, vcn_id=vcn.id, display_name="dmb-subnet-jobs").data
+                if s.lifecycle_state == "AVAILABLE"), None)
+    if sub is None:
+        sub = vn.create_subnet(oci.core.models.CreateSubnetDetails(
+            compartment_id=cid, vcn_id=vcn.id, display_name="dmb-subnet-jobs", cidr_block=SUBNET_CIDR,
+            dns_label="jobs", prohibit_public_ip_on_vnic=True, route_table_id=rt.id,
+            security_list_ids=[sl.id], freeform_tags=TAGS)).data
+        sub = comp(vn, vn.get_subnet(sub.id), "lifecycle_state", "AVAILABLE").data
+    record("Subnet (privada)", f"dmb-subnet-jobs ({SUBNET_CIDR})", sub.id, REGION)
+
+    # O NAT é criado só por nat.py; aqui ele só é registrado, se existir.
+    for nat in vn.list_nat_gateways(cid, vcn_id=vcn.id).data:
+        if nat.lifecycle_state == "AVAILABLE":
+            status = "bloqueado" if nat.block_traffic else "ATIVO"
+            record("NAT Gateway (temporário)", f"{nat.display_name} ({status})", nat.id, REGION)
+
+
+USER_FAMILIES = [
+    "data-science-family",
+    "object-family",
+    "virtual-network-family",
+    "logging-family",
+    "repos",
+    "generative-ai-family",
+]
+
+
+def ensure_policy(idn, cid, name, description, statements):
+    pol = next((p for p in idn.list_policies(cid).data if p.name == name), None)
+    if pol is None:
+        pol = idn.create_policy(oci.identity.models.CreatePolicyDetails(
+            compartment_id=cid, name=name, description=description,
+            statements=statements, freeform_tags=TAGS)).data
+    elif sorted(pol.statements) != sorted(statements):
+        pol = idn.update_policy(pol.id, oci.identity.models.UpdatePolicyDetails(
+            description=description, statements=statements)).data
+    record("IAM Policy", f"{name} ({len(statements)} declarações)", pol.id, "global (home us-ashburn-1)")
+
+
+def ensure_policies(idn, cid, user_id):
+    # Sem acesso de administrador à tenancy: nada de grupos nem dynamic groups.
+    # O usuário é identificado pelo OCID e os jobs pelo tipo de principal.
+    comp = f"compartment id {cid}"
+    user_stmts = [f"Allow any-user to manage {fam} in {comp} where request.user.id = '{user_id}'"
+                  for fam in USER_FAMILIES]
+    ensure_policy(idn, cid, "dmb-usuario-manager",
+                  "Usuário do benchmark como manager das famílias usadas", user_stmts)
+
+    job = f"request.principal.type = 'datasciencejob', request.principal.compartment.id = '{cid}'"
+    read_buckets = "any {target.bucket.name = 'dmb-entrada', target.bucket.name = 'dmb-artefatos'}"
+    write_buckets = "any {target.bucket.name = 'dmb-resultados', target.bucket.name = 'dmb-logs-imutaveis'}"
+    job_stmts = [
+        f"Allow any-user to read buckets in {comp} where all {{{job}}}",
+        f"Allow any-user to read objects in {comp} where all {{{job}, {read_buckets}}}",
+        f"Allow any-user to manage objects in {comp} where all {{{job}, {write_buckets}, "
+        "any {request.permission = 'OBJECT_CREATE', request.permission = 'OBJECT_INSPECT', request.permission = 'OBJECT_READ'}}",
+        f"Allow any-user to use log-content in {comp} where all {{{job}}}",
+        f"Allow any-user to read repos in {comp} where all {{{job}}}",
+        f"Allow service datascience to use virtual-network-family in {comp}",
+    ]
+    ensure_policy(idn, cid, "dmb-jobs-menor-privilegio",
+                  "Jobs de benchmark: leitura de entrada/artefatos e escrita de resultados/logs (tarefa 7)",
+                  job_stmts)
+
+
+def write_artifacts():
+    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+    lines = [
+        "# Artefatos OCI — benchmark de modelos de decisão",
+        "",
+        f"Atualizado em {now} por `infra/oci/provision.py`. Região dos recursos: `{REGION}`.",
+        f"Tags livres aplicadas: {', '.join(f'`{k}={v}`' for k, v in TAGS.items())} (`centro_custo` pendente, Q13).",
+        "",
+        "| Tipo | Nome | OCID | Região |",
+        "|------|------|------|--------|",
+    ]
+    lines += [f"| {k} | {n} | `{o}` | {r} |" for k, n, o, r in artifacts]
+    ARTIFACTS_FILE.write_text("\n".join(lines) + "\n")
+
+
+def main():
+    home = load_config(region="us-ashburn-1")
+    cfg = load_config()
+    idn = oci.identity.IdentityClient(home)
+    cid = find_compartment(idn, home["tenancy"])
+    ensure_policies(idn, cid, home["user"])
+    ensure_ds_project(cfg, cid)
+    ensure_buckets(cfg, cid)
+    ensure_network(cfg, cid)
+    write_artifacts()
+    for a in artifacts:
+        print(" | ".join(a))
+
+
+if __name__ == "__main__":
+    main()

infra/oci/session.py

@@ -0,0 +1,25 @@
+"""Sessão OCI do benchmark a partir das credenciais em .secrets/oci-credentials/."""
+import configparser
+from pathlib import Path
+
+import oci
+
+ROOT = Path(__file__).resolve().parents[2]
+CRED_DIR = ROOT / ".secrets" / "oci-credentials"
+REGION = "us-chicago-1"
+
+
+def load_config(region: str = REGION, profile: str = "DEFAULT") -> dict:
+    parser = configparser.ConfigParser()
+    parser.read(CRED_DIR / "profile.config")
+    cfg = dict(parser[profile])
+    key_file = cfg.get("key_file", "").split("#")[0].strip()
+    if not key_file or not Path(key_file).expanduser().is_file():
+        keys = [p for p in CRED_DIR.glob("*.pem") if not p.name.endswith("_public.pem")]
+        if len(keys) != 1:
+            raise RuntimeError(f"esperada 1 chave privada em {CRED_DIR}, encontradas {len(keys)}")
+        key_file = str(keys[0])
+    cfg["key_file"] = key_file
+    cfg["region"] = region
+    oci.config.validate_config(cfg)
+    return cfg