setup models

2026-11-06 06:14:11 +00:00 · 2026-10-07 13:43:09 -03:00
parent aeee6c7de1
commit 84ea9a5307
75 changed files with 5093 additions and 58 deletions
--- a/infra/oci/provision.py
+++ b/infra/oci/provision.py
@@ -1,8 +1,13 @@
-"""Provisionamento idempotente da fase F0 (tarefas 1, 2, 5, 6, 7, 8).
+"""Provisionamento idempotente da fase F0 (tarefas 1, 2, 5, 6, 7, 8) e do ambiente de execução.
+
+Ambiente de execução: a máquina local é só IDE e repositório. Os scripts rodam na VM
+dmb-vm (sub-rede de desenvolvimento, SSH só do IP autorizado) e em notebooks e jobs do
+OCI Data Science. A sub-rede privada dmb-subnet-jobs continua sem saída para a internet.

 Uso: PYTHONPATH=infra/oci .venv/bin/python infra/oci/provision.py
 Registra os artefatos criados em .secrets/artefatos-OCI.md.
 """
+import os
 import time
 from datetime import datetime, timezone

@@ -16,7 +21,19 @@ TAGS = {"projeto": "benchmark-decisoes-ptbr", "ambiente": "benchmark"}  # centro
 BUCKETS = ["entrada", "artefatos", "resultados", "logs-imutaveis"]
 VCN_CIDR = "10.20.0.0/16"
 SUBNET_CIDR = "10.20.1.0/24"
+DEV_SUBNET_CIDR = "10.20.2.0/24"
+SSH_ALLOWED_CIDR = os.environ.get("DMB_SSH_CIDR", "203.0.113.1/32")
+# VM de trabalho criada pelo usuário no console (o lançamento via API é negado neste compartment).
+VM_NAME = "instance-20261007-1206"
+VM_SSH_KEY = ".secrets/ssh/ssh-key-oci.key"
+VM_USER = "opc"
+NOTEBOOK_NAME = "dmb-notebook-cpu"
+NOTEBOOK_SHAPE = ("VM.Standard.E4.Flex", 4, 32)
 ARTIFACTS_FILE = CRED_DIR.parent / "artefatos-OCI.md"
+# Região secundária só para jobs de GPU (A10 sem capacidade em us-chicago-1).
+GPU_REGION = "sa-saopaulo-1"
+GPU_SUFFIX = "-gru"
+GPU_BUCKETS = ["artefatos", "resultados"]

 artifacts = []

@@ -64,21 +81,22 @@ def ensure_ds_project(cfg, cid):
        proj = ds.create_project(oci.data_science.models.CreateProjectDetails(
            compartment_id=cid, display_name=name, freeform_tags=TAGS,
            description="Benchmark de modelos de decisão PT-BR")).data
-    record("Data Science Project", name, proj.id, REGION)
+    record("Data Science Project", name, proj.id, cfg["region"])
+    return proj.id


-def ensure_buckets(cfg, cid):
+def ensure_buckets(cfg, cid, suffix="", buckets=BUCKETS):
    os_ = oci.object_storage.ObjectStorageClient(cfg)
    ns = os_.get_namespace().data
    existing = {b.name for b in oci.pagination.list_call_get_all_results(os_.list_buckets, ns, cid).data}
-    for short in BUCKETS:
-        name = f"dmb-{short}"
+    for short in buckets:
+        name = f"dmb-{short}{suffix}"
        if name not in existing:
            os_.create_bucket(ns, oci.object_storage.models.CreateBucketDetails(
                name=name, compartment_id=cid, public_access_type="NoPublicAccess",
                storage_tier="Standard", versioning="Enabled", freeform_tags=TAGS))
        b = os_.get_bucket(ns, name).data
-        record("Object Storage Bucket", f"{name} (namespace {ns})", b.id, REGION)
+        record("Object Storage Bucket", f"{name} (namespace {ns})", b.id, cfg["region"])


 def ensure_network(cfg, cid):
@@ -92,7 +110,7 @@ def ensure_network(cfg, cid):
            compartment_id=cid, display_name="dmb-vcn", cidr_blocks=[VCN_CIDR],
            dns_label="dmbvcn", freeform_tags=TAGS)).data
        vcn = comp(vn, vn.get_vcn(vcn.id), "lifecycle_state", "AVAILABLE").data
-    record("VCN", f"dmb-vcn ({VCN_CIDR})", vcn.id, REGION)
+    record("VCN", f"dmb-vcn ({VCN_CIDR})", vcn.id, cfg["region"])

    svc = next(s for s in vn.list_services().data if s.cidr_block.startswith("all-") and "services-in-oracle-services-network" in s.cidr_block)
    sgw = next((g for g in vn.list_service_gateways(cid, vcn_id=vcn.id).data
@@ -102,7 +120,7 @@ def ensure_network(cfg, cid):
            compartment_id=cid, vcn_id=vcn.id, display_name="dmb-sgw", freeform_tags=TAGS,
            services=[oci.core.models.ServiceIdRequestDetails(service_id=svc.id)])).data
        sgw = comp(vn, vn.get_service_gateway(sgw.id), "lifecycle_state", "AVAILABLE").data
-    record("Service Gateway", f"dmb-sgw ({svc.cidr_block})", sgw.id, REGION)
+    record("Service Gateway", f"dmb-sgw ({svc.cidr_block})", sgw.id, cfg["region"])

    rt = next((r for r in vn.list_route_tables(cid, vcn_id=vcn.id, display_name="dmb-rt-privada").data
               if r.lifecycle_state == "AVAILABLE"), None)
@@ -113,7 +131,7 @@ def ensure_network(cfg, cid):
                destination=svc.cidr_block, destination_type="SERVICE_CIDR_BLOCK",
                network_entity_id=sgw.id, description="Object Storage e serviços OCI via Service Gateway")])).data
        rt = comp(vn, vn.get_route_table(rt.id), "lifecycle_state", "AVAILABLE").data
-    record("Route Table", "dmb-rt-privada (somente Service Gateway)", rt.id, REGION)
+    record("Route Table", "dmb-rt-privada (somente Service Gateway)", rt.id, cfg["region"])

    sl = next((s for s in vn.list_security_lists(cid, vcn_id=vcn.id, display_name="dmb-sl-privada").data
               if s.lifecycle_state == "AVAILABLE"), None)
@@ -131,7 +149,7 @@ def ensure_network(cfg, cid):
                oci.core.models.IngressSecurityRule(source=VCN_CIDR, protocol="all", description="Tráfego interno da VCN"),
            ])).data
        sl = comp(vn, vn.get_security_list(sl.id), "lifecycle_state", "AVAILABLE").data
-    record("Security List", "dmb-sl-privada (egresso só para serviços OCI e VCN)", sl.id, REGION)
+    record("Security List", "dmb-sl-privada (egresso só para serviços OCI e VCN)", sl.id, cfg["region"])

    sub = next((s for s in vn.list_subnets(cid, vcn_id=vcn.id, display_name="dmb-subnet-jobs").data
                if s.lifecycle_state == "AVAILABLE"), None)
@@ -141,13 +159,14 @@ def ensure_network(cfg, cid):
            dns_label="jobs", prohibit_public_ip_on_vnic=True, route_table_id=rt.id,
            security_list_ids=[sl.id], freeform_tags=TAGS)).data
        sub = comp(vn, vn.get_subnet(sub.id), "lifecycle_state", "AVAILABLE").data
-    record("Subnet (privada)", f"dmb-subnet-jobs ({SUBNET_CIDR})", sub.id, REGION)
+    record("Subnet (privada)", f"dmb-subnet-jobs ({SUBNET_CIDR})", sub.id, cfg["region"])

    # O NAT é criado só por nat.py; aqui ele só é registrado, se existir.
    for nat in vn.list_nat_gateways(cid, vcn_id=vcn.id).data:
        if nat.lifecycle_state == "AVAILABLE":
            status = "bloqueado" if nat.block_traffic else "ATIVO"
-            record("NAT Gateway (temporário)", f"{nat.display_name} ({status})", nat.id, REGION)
+            record("NAT Gateway (temporário)", f"{nat.display_name} ({status})", nat.id, cfg["region"])
+    return vcn.id


 USER_FAMILIES = [
@@ -157,6 +176,9 @@ USER_FAMILIES = [
    "logging-family",
    "repos",
    "generative-ai-family",
+    "instance-family",
+    "volume-family",
+    "instance-agent-command-family",
 ]


@@ -192,18 +214,130 @@ def ensure_policies(idn, cid, user_id):
        f"Allow any-user to use log-content in {comp} where all {{{job}}}",
        f"Allow any-user to read repos in {comp} where all {{{job}}}",
        f"Allow service datascience to use virtual-network-family in {comp}",
+        # copy_object entre regiões (réplica para a região de GPU)
+        f"Allow service objectstorage-{REGION} to manage object-family in {comp}",
+        f"Allow service objectstorage-{GPU_REGION} to manage object-family in {comp}",
    ]
    ensure_policy(idn, cid, "dmb-jobs-menor-privilegio",
                  "Jobs de benchmark: leitura de entrada/artefatos e escrita de resultados/logs (tarefa 7)",
                  job_stmts)


+def ensure_workload_policy(idn, cid):
+    """VM (instance principal) e notebooks do Data Science (resource principal), sem dynamic groups."""
+    comp = f"compartment id {cid}"
+    who = (f"all {{request.principal.compartment.id = '{cid}', "
+           "any {request.principal.type = 'instance', request.principal.type = 'datasciencenotebooksession'}}")
+    stmts = [f"Allow any-user to {verb} in {comp} where {who}" for verb in (
+        "read buckets",
+        "manage objects",
+        "manage data-science-family",
+        "manage repos",
+        "use virtual-network-family",
+        "use log-groups",
+        "use log-content",
+        "use generative-ai-family",
+        "read compartments",
+    )]
+    ensure_policy(idn, cid, "dmb-workloads", "VM de trabalho e notebooks do benchmark", stmts)
+
+
+def ensure_dev_network(vn, cid, vcn_id):
+    cfg = {"region": REGION}  # a sub-rede de desenvolvimento só existe na região principal
+    igw = next((g for g in vn.list_internet_gateways(cid, vcn_id=vcn_id).data
+                if g.lifecycle_state == "AVAILABLE"), None)
+    if igw is None:
+        igw = vn.create_internet_gateway(oci.core.models.CreateInternetGatewayDetails(
+            compartment_id=cid, vcn_id=vcn_id, display_name="dmb-igw-dev", is_enabled=True,
+            freeform_tags=TAGS)).data
+        igw = oci.wait_until(vn, vn.get_internet_gateway(igw.id), "lifecycle_state", "AVAILABLE").data
+    record("Internet Gateway", "dmb-igw-dev (só para a sub-rede de desenvolvimento)", igw.id, cfg["region"])
+
+    rt = next((r for r in vn.list_route_tables(cid, vcn_id=vcn_id, display_name="dmb-rt-dev").data
+               if r.lifecycle_state == "AVAILABLE"), None)
+    if rt is None:
+        rt = vn.create_route_table(oci.core.models.CreateRouteTableDetails(
+            compartment_id=cid, vcn_id=vcn_id, display_name="dmb-rt-dev", freeform_tags=TAGS,
+            route_rules=[oci.core.models.RouteRule(destination="0.0.0.0/0", destination_type="CIDR_BLOCK",
+                                                   network_entity_id=igw.id)])).data
+        rt = oci.wait_until(vn, vn.get_route_table(rt.id), "lifecycle_state", "AVAILABLE").data
+    record("Route Table", "dmb-rt-dev (internet via IGW)", rt.id, cfg["region"])
+
+    ingress = [oci.core.models.IngressSecurityRule(
+        source=SSH_ALLOWED_CIDR, protocol="6", description="SSH só do IP autorizado",
+        tcp_options=oci.core.models.TcpOptions(destination_port_range=oci.core.models.PortRange(min=22, max=22)))]
+    sl = next((x for x in vn.list_security_lists(cid, vcn_id=vcn_id, display_name="dmb-sl-dev").data
+               if x.lifecycle_state == "AVAILABLE"), None)
+    if sl is None:
+        sl = vn.create_security_list(oci.core.models.CreateSecurityListDetails(
+            compartment_id=cid, vcn_id=vcn_id, display_name="dmb-sl-dev", freeform_tags=TAGS,
+            ingress_security_rules=ingress,
+            egress_security_rules=[oci.core.models.EgressSecurityRule(destination="0.0.0.0/0", protocol="all")])).data
+        sl = oci.wait_until(vn, vn.get_security_list(sl.id), "lifecycle_state", "AVAILABLE").data
+    elif [r.source for r in sl.ingress_security_rules] != [SSH_ALLOWED_CIDR]:
+        vn.update_security_list(sl.id, oci.core.models.UpdateSecurityListDetails(
+            ingress_security_rules=ingress, egress_security_rules=sl.egress_security_rules))
+    record("Security List", f"dmb-sl-dev (SSH de {SSH_ALLOWED_CIDR})", sl.id, cfg["region"])
+
+    sub = next((x for x in vn.list_subnets(cid, vcn_id=vcn_id, display_name="dmb-subnet-dev").data
+                if x.lifecycle_state == "AVAILABLE"), None)
+    if sub is None:
+        sub = vn.create_subnet(oci.core.models.CreateSubnetDetails(
+            compartment_id=cid, vcn_id=vcn_id, display_name="dmb-subnet-dev", cidr_block=DEV_SUBNET_CIDR,
+            dns_label="dev", prohibit_public_ip_on_vnic=False, route_table_id=rt.id,
+            security_list_ids=[sl.id], freeform_tags=TAGS)).data
+        sub = oci.wait_until(vn, vn.get_subnet(sub.id), "lifecycle_state", "AVAILABLE").data
+    record("Subnet (desenvolvimento)", f"dmb-subnet-dev ({DEV_SUBNET_CIDR})", sub.id, cfg["region"])
+    return sub.id
+
+
+def ensure_vm(cfg, cid):
+    """Registra a VM de trabalho, os volumes anexados e o acesso SSH. Não cria instâncias."""
+    cc = oci.core.ComputeClient(cfg)
+    vm = next((i for i in cc.list_instances(cid, display_name=VM_NAME).data
+               if i.lifecycle_state not in ("TERMINATED", "TERMINATING")), None)
+    if vm is None:
+        print(f"aviso: VM {VM_NAME} não encontrada")
+        return None
+    vn = oci.core.VirtualNetworkClient(cfg)
+    bs = oci.core.BlockstorageClient(cfg)
+    vnic = vn.get_vnic(cc.list_vnic_attachments(cid, instance_id=vm.id).data[0].vnic_id).data
+    record("Compute Instance", f"{VM_NAME} ({vm.shape}, {vm.shape_config.ocpus:.0f} OCPU, "
+           f"{vm.shape_config.memory_in_gbs:.0f} GB, Oracle Linux 9, IP público {vnic.public_ip}, "
+           f"usuário {VM_USER}, chave {VM_SSH_KEY})", vm.id, cfg["region"])
+    for att in cc.list_boot_volume_attachments(vm.availability_domain, cid, instance_id=vm.id).data:
+        bv = bs.get_boot_volume(att.boot_volume_id).data
+        record("Boot Volume", f"{bv.display_name} ({bv.size_in_gbs} GB, raiz /)", bv.id, cfg["region"])
+    for att in cc.list_volume_attachments(cid, instance_id=vm.id).data:
+        if att.lifecycle_state == "ATTACHED":
+            v = bs.get_volume(att.volume_id).data
+            record("Block Volume", f"{v.display_name} ({v.size_in_gbs} GB, {att.attachment_type}, "
+                   f"{att.device}, montado em /data)", v.id, cfg["region"])
+    return vnic.public_ip
+
+
+def ensure_notebook(cfg, cid, project_id):
+    ds = oci.data_science.DataScienceClient(cfg)
+    nb = next((n for n in ds.list_notebook_sessions(cid, project_id=project_id).data
+               if n.display_name == NOTEBOOK_NAME and n.lifecycle_state not in ("DELETED", "DELETING")), None)
+    if nb is None:
+        shape, ocpus, mem = NOTEBOOK_SHAPE
+        nb = ds.create_notebook_session(oci.data_science.models.CreateNotebookSessionDetails(
+            compartment_id=cid, project_id=project_id, display_name=NOTEBOOK_NAME, freeform_tags=TAGS,
+            notebook_session_config_details=oci.data_science.models.NotebookSessionConfigDetails(
+                shape=shape, block_storage_size_in_gbs=200,
+                notebook_session_shape_config_details=oci.data_science.models.NotebookSessionShapeConfigDetails(
+                    ocpus=ocpus, memory_in_gbs=mem)))).data
+    record("Data Science Notebook Session", f"{NOTEBOOK_NAME} ({NOTEBOOK_SHAPE[0]}, rede gerenciada, "
+           f"estado {nb.lifecycle_state})", nb.id, cfg["region"])
+
+
 def write_artifacts():
    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
    lines = [
        "# Artefatos OCI — benchmark de modelos de decisão",
        "",
-        f"Atualizado em {now} por `infra/oci/provision.py`. Região dos recursos: `{REGION}`.",
+        f"Atualizado em {now} por `infra/oci/provision.py`. Região principal: `{REGION}`; GPU também em `{GPU_REGION}`.",
        f"Tags livres aplicadas: {', '.join(f'`{k}={v}`' for k, v in TAGS.items())} (`centro_custo` pendente, Q13).",
        "",
        "| Tipo | Nome | OCID | Região |",
@@ -219,9 +353,24 @@ def main():
    idn = oci.identity.IdentityClient(home)
    cid = find_compartment(idn, home["tenancy"])
    ensure_policies(idn, cid, home["user"])
-    ensure_ds_project(cfg, cid)
+    ensure_workload_policy(idn, cid)
+    project_id = ensure_ds_project(cfg, cid)
    ensure_buckets(cfg, cid)
-    ensure_network(cfg, cid)
+    vcn_id = ensure_network(cfg, cid)
+    ensure_dev_network(oci.core.VirtualNetworkClient(cfg), cid, vcn_id)
+    ensure_vm(cfg, cid)
+    ensure_notebook(cfg, cid, project_id)
+
+    gru = load_config(region=GPU_REGION)
+    ensure_ds_project(gru, cid)
+    ensure_buckets(gru, cid, suffix=GPU_SUFFIX, buckets=GPU_BUCKETS)
+    try:
+        ensure_network(gru, cid)
+    except oci.exceptions.ServiceError as error:
+        if error.code != "LimitExceeded":
+            raise
+        # Sem VCN disponível na região: jobs de GPU usam rede gerenciada + isolamento no processo.
+        print(f"aviso: VCN indisponível em {GPU_REGION} ({error.message[:80]})")
    write_artifacts()
    for a in artifacts:
        print(" | ".join(a))
--- a/infra/oci/replicate.py
+++ b/infra/oci/replicate.py
@@ -0,0 +1,63 @@
+"""Réplica server-side dos artefatos para a região de GPU (copy_object entre regiões).
+
+Copia os pesos (models/) e o ambiente atual de cada candidato (envs/<c>/latest.json e o
+pacote que ele referencia). Objetos já presentes com o mesmo tamanho são pulados.
+
+Uso: PYTHONPATH=infra/oci .venv/bin/python infra/oci/replicate.py [--wait]
+"""
+import json
+import sys
+import time
+
+import oci
+
+import provision
+from session import load_config
+
+SRC_BUCKET = "dmb-artefatos"
+DST_BUCKET = f"dmb-artefatos{provision.GPU_SUFFIX}"
+
+
+def main():
+    src = oci.object_storage.ObjectStorageClient(load_config())
+    dst = oci.object_storage.ObjectStorageClient(load_config(region=provision.GPU_REGION))
+    ns = src.get_namespace().data
+    objects = oci.pagination.list_call_get_all_results(
+        src.list_objects, ns, SRC_BUCKET, fields="size").data.objects
+    sizes = {o.name: o.size for o in objects}
+    wanted = [n for n in sizes if n.startswith("models/")]
+    for cand in ("laya", "gliner", "semif", "rizzo_flow"):
+        latest = f"envs/{cand}/latest.json"
+        if latest in sizes:
+            obj = json.loads(src.get_object(ns, SRC_BUCKET, latest).data.content)["object"]
+            wanted += [latest, obj, obj.replace(".tar.gz", "-env-manifest.json")]
+    cuda = "runtimes/rizzo_flow/cuda-latest.json"
+    if cuda in sizes:
+        wanted += [cuda, json.loads(src.get_object(ns, SRC_BUCKET, cuda).data.content)["object"]]
+    existing = {o.name: o.size for o in oci.pagination.list_call_get_all_results(
+        dst.list_objects, ns, DST_BUCKET, fields="size").data.objects}
+    requests = []
+    for name in wanted:
+        if name not in sizes:
+            continue
+        if existing.get(name) == sizes[name] and not name.endswith("latest.json"):
+            continue
+        r = src.copy_object(ns, SRC_BUCKET, oci.object_storage.models.CopyObjectDetails(
+            source_object_name=name, destination_region=provision.GPU_REGION,
+            destination_namespace=ns, destination_bucket=DST_BUCKET, destination_object_name=name))
+        requests.append((name, r.headers["opc-work-request-id"]))
+    total = sum(sizes[n] for n, _ in requests)
+    print(f"{len(requests)} cópias iniciadas ({total / 1e9:.2f} GB) para {DST_BUCKET} em {provision.GPU_REGION}")
+    if "--wait" in sys.argv:
+        pending = dict(requests)
+        while pending:
+            for name, wr in list(pending.items()):
+                status = src.get_work_request(wr).data.status
+                if status in ("COMPLETED", "FAILED", "CANCELED"):
+                    print(status, name, flush=True)
+                    pending.pop(name)
+            time.sleep(20)
+
+
+if __name__ == "__main__":
+    main()