setup models

infra/ds/build_env.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Job de build (rede gerenciada, com internet): cria o ambiente Python do candidato com as
+# versões fixadas de env/requirements/, empacota e publica em dmb-artefatos/envs/<candidato>/.
+# Os jobs de benchmark (sub-rede privada) só baixam esse pacote; nunca instalam da internet.
+set -euo pipefail
+CAND="${DMB_CANDIDATE:?defina DMB_CANDIDATE}"
+CODE="$(cd "$(dirname "$0")/../.." && pwd)"
+BASE_PY=/opt/conda/bin/python3
+ENV_DIR="$HOME/envs/$CAND"
+STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+cd "$CODE"
+
+echo "== build $CAND em $ENV_DIR (código $DMB_CODE_REV)"
+rm -rf "$ENV_DIR"
+"$BASE_PY" -m venv "$ENV_DIR"
+"$ENV_DIR/bin/pip" install -q --upgrade "pip==25.2"
+"$ENV_DIR/bin/pip" install -q -r env/requirements/common.txt "pytest==8.4.2" "httpx==0.28.1"
+"$ENV_DIR/bin/pip" install -q --extra-index-url https://download.pytorch.org/whl/cu128 -r "env/requirements/$CAND.txt"
+
+src_object() {  # objeto do tarball de código-fonte do candidato, segundo o manifesto
+  "$BASE_PY" -c "import json,sys; m=json.load(open('env/artifacts-manifest.json')); print(next(a['object'] for a in m['artifacts'] if a['candidate']=='$CAND' and a['kind']=='source'))"
+}
+case "$CAND" in
+  semif|rizzo_flow)
+    OBJ="$(src_object)"
+    "$BASE_PY" -m harness.objstore get dmb-artefatos "$OBJ" /tmp/src.tar.gz
+    mkdir -p /tmp/src && tar xzf /tmp/src.tar.gz -C /tmp/src
+    "$ENV_DIR/bin/pip" install -q --no-deps /tmp/src/*/
+    ;;
+esac
+if [ "$CAND" = rizzo_flow ]; then
+  # Runtime llama.cpp b11081 (CPU e CUDA), com o SHA-256 conferido pelo próprio Rizzo Flow.
+  (cd "$ENV_DIR" && "$ENV_DIR/bin/rizzo" download --only runtime --runtime cpu \
+     && "$ENV_DIR/bin/rizzo" download --only runtime --runtime cuda && rm -rf runtimes/downloads)
+fi
+
+"$ENV_DIR/bin/python" -m harness.image_manifest > "$ENV_DIR/env-manifest.json"
+"$ENV_DIR/bin/python" -m pytest -q tests 2>&1 | tail -2
+
+TARBALL="/tmp/$CAND-env.tar.gz"
+tar czf "$TARBALL" -C "$HOME" "envs/$CAND"
+OBJECT="envs/$CAND/$STAMP.tar.gz"
+SHA="$("$BASE_PY" -m harness.objstore put dmb-artefatos "$OBJECT" "$TARBALL" --sha256)"
+SIZE="$(stat -c %s "$TARBALL")"
+cat > /tmp/latest.json <<JSON
+{"candidate": "$CAND", "object": "$OBJECT", "sha256": "$SHA", "bytes": $SIZE,
+ "built_at": "$STAMP", "code_rev": "${DMB_CODE_REV:-}", "python": "$("$ENV_DIR/bin/python" --version 2>&1)",
+ "home": "$HOME"}
+JSON
+"$BASE_PY" -m harness.objstore put dmb-artefatos "envs/$CAND/latest.json" /tmp/latest.json
+"$BASE_PY" -m harness.objstore put dmb-artefatos "envs/$CAND/$STAMP-env-manifest.json" "$ENV_DIR/env-manifest.json"
+cat /tmp/latest.json
+echo "== ok"

infra/ds/build_llama_cuda.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Job de build (rede gerenciada): compila o llama.cpp com CUDA para a imagem dos jobs do Data Science.
+# Os binários oficiais CUDA do llama.cpp exigem GLIBC 2.38, ausente nessa imagem. Compilamos o mesmo
+# commit fixado pelo Rizzo Flow, com sysroot glibc 2.17, libstdc++ estática e CUDA 12.8 (sm_80/sm_86),
+# no mesmo layout dos pacotes oficiais (backends como plug-ins, libs CUDA no mesmo diretório).
+set -euo pipefail
+COMMIT=161755f29e415e2c33efe906e91843c068efd664   # rizzo_flow.llama_release.COMMIT (b11081)
+CODE="$(cd "$(dirname "$0")/../.." && pwd)"
+BASE_PY=/opt/conda/bin/python3
+TC="$HOME/toolchain"
+OUT_NAME="llama-b11081-linux-x64-cuda-dsjobs"
+OUT="$HOME/$OUT_NAME"
+STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+
+echo "== glibc da imagem: $(ldd --version | head -1)"
+/opt/conda/bin/conda create -q -y -p "$TC" -c conda-forge \
+  "cuda-nvcc=12.8" "cuda-cudart-dev=12.8" "libcublas-dev=12.8" "cuda-version=12.8" \
+  cmake ninja git "gxx_linux-64=13" "sysroot_linux-64=2.17" >/dev/null
+export PATH="$TC/bin:$PATH" CUDAToolkit_ROOT="$TC" CUDACXX="$TC/bin/nvcc"
+export CC="$TC/bin/x86_64-conda-linux-gnu-gcc" CXX="$TC/bin/x86_64-conda-linux-gnu-g++"
+
+git clone -q https://github.com/ggml-org/llama.cpp /tmp/llama.cpp
+git -C /tmp/llama.cpp checkout -q "$COMMIT"
+cmake -S /tmp/llama.cpp -B /tmp/build -G Ninja -DCMAKE_BUILD_TYPE=Release \
+  -DBUILD_SHARED_LIBS=ON -DGGML_BACKEND_DL=ON -DGGML_NATIVE=OFF -DGGML_CPU_ALL_VARIANTS=ON \
+  -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="80;86" -DLLAMA_CURL=OFF \
+  -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_SERVER=OFF -DLLAMA_BUILD_TOOLS=OFF \
+  -DCMAKE_SHARED_LINKER_FLAGS="-static-libstdc++ -static-libgcc" \
+  -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON -DCMAKE_INSTALL_RPATH='$ORIGIN' -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+  >/tmp/cmake.log
+cmake --build /tmp/build -j "$(nproc)" >/tmp/build.log 2>&1 || { tail -50 /tmp/build.log; exit 1; }
+
+rm -rf "$OUT" && mkdir -p "$OUT"
+find /tmp/build/bin -maxdepth 1 -name "*.so" -exec cp -L {} "$OUT/" \;
+for lib in libcudart.so.12 libcublas.so.12 libcublasLt.so.12; do cp -L "$TC/lib/$lib" "$OUT/"; done
+ls -la "$OUT"
+echo "== dependências não resolvidas:"; for f in "$OUT"/*.so*; do LD_LIBRARY_PATH="$OUT" ldd "$f" | grep "not found" || true; done
+echo "== maior GLIBC exigida:"; objdump -T "$OUT"/*.so* 2>/dev/null | grep -o "GLIBC_[0-9.]*" | sort -uV | tail -1
+
+cd "$CODE"
+TARBALL="/tmp/$OUT_NAME.tar.gz"
+tar czf "$TARBALL" -C "$HOME" "$OUT_NAME"
+OBJECT="runtimes/rizzo_flow/$OUT_NAME-$STAMP.tar.gz"
+SHA="$("$BASE_PY" -m harness.objstore put dmb-artefatos "$OBJECT" "$TARBALL" --sha256)"
+cat > /tmp/cuda-latest.json <<JSON
+{"object": "$OBJECT", "sha256": "$SHA", "dir": "$OUT_NAME", "llama_commit": "$COMMIT",
+ "cuda": "12.8", "archs": "80;86", "built_at": "$STAMP", "code_rev": "${DMB_CODE_REV:-}"}
+JSON
+"$BASE_PY" -m harness.objstore put dmb-artefatos runtimes/rizzo_flow/cuda-latest.json /tmp/cuda-latest.json
+cat /tmp/cuda-latest.json
+echo "== ok"

infra/ds/gpu_probe.py

@@ -0,0 +1,37 @@
+"""Encontra um shape de GPU com capacidade: roda o job em cada shape, em ordem, até um não falhar
+por falta de capacidade. Imprime o shape usado e o estado final.
+
+Uso: PYTHONPATH=infra/oci:infra/ds .venv/bin/python infra/ds/gpu_probe.py <candidato> [shape ...]
+"""
+import sys
+
+import jobs
+
+DEFAULT_SHAPES = ["VM.GPU.A10.1", "VM.GPU.A10.2", "VM.GPU.A100.B40G.1", "VM.GPU.A100.80G.1", "VM.GPU3.1"]
+
+
+def main():
+    args = sys.argv[1:]
+    region = None
+    if args and args[0].startswith("--region="):
+        region = args.pop(0).split("=", 1)[1]
+    candidate, shapes = args[0], args[1:] or DEFAULT_SHAPES
+    for shape in shapes:
+        try:
+            run_id = jobs.run_job(f"sanity-{candidate}-gpu", "infra/ds/run_job.sh", shape, None, None, region is None,
+                                  {"DMB_CANDIDATE": candidate, "DMB_MODE": "sanity", "DMB_DEVICE": "auto"},
+                                  storage_gb=60, region=region)
+        except jobs.oci.exceptions.ServiceError as error:
+            print(f"indisponível: {shape} ({error.code})", flush=True)
+            continue
+        r = jobs.wait(run_id, poll=20)
+        if "no capacity" in (r.lifecycle_details or "").lower():
+            print(f"sem capacidade: {shape}", flush=True)
+            continue
+        print(f"SHAPE {shape} -> {r.lifecycle_state} {run_id}", flush=True)
+        return
+    print("nenhum shape com capacidade", flush=True)
+
+
+if __name__ == "__main__":
+    main()

infra/ds/gpu_queue.py

@@ -0,0 +1,47 @@
+"""Fila de jobs de GPU (o limite em sa-saopaulo-1 é de uma A10 por vez).
+
+Uso: PYTHONPATH=infra/oci:infra/ds .venv/bin/python infra/ds/gpu_queue.py <modo> <candidato> [...] \
+        [--after <job_run_ocid>] [--shape VM.GPU.A10.1]
+Grava "candidato run_ocid estado" em runs/gpu-queue.log.
+"""
+import argparse
+import time
+from pathlib import Path
+
+import jobs
+import provision
+
+LOG = Path(__file__).resolve().parents[2] / "runs" / "gpu-queue.log"
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("mode")
+    ap.add_argument("candidates", nargs="+")
+    ap.add_argument("--after")
+    ap.add_argument("--shape", default="VM.GPU.A10.1")
+    ap.add_argument("--storage", type=int, default=100)
+    args = ap.parse_args()
+    LOG.parent.mkdir(exist_ok=True)
+    if args.after:
+        jobs.wait(args.after, poll=30)
+    for cand in args.candidates:
+        while True:
+            try:
+                run_id = jobs.run_job(f"{args.mode}-{cand}-gpu", "infra/ds/run_job.sh", args.shape, None, None,
+                                      False, {"DMB_CANDIDATE": cand, "DMB_MODE": args.mode, "DMB_DEVICE": "auto"},
+                                      storage_gb=args.storage, region=provision.GPU_REGION)
+                break
+            except jobs.oci.exceptions.ServiceError as error:
+                if error.code != "LimitExceeded":
+                    raise
+                time.sleep(60)  # outra GPU ainda ocupada
+        r = jobs.wait(run_id, poll=30)
+        line = f"{cand} {run_id} {r.lifecycle_state} {(r.lifecycle_details or '')[:80]}"
+        print(line, flush=True)
+        with LOG.open("a") as f:
+            f.write(line + "\n")
+
+
+if __name__ == "__main__":
+    main()

infra/ds/jobs.py

@@ -0,0 +1,187 @@
+"""Execução no OCI Data Science: a máquina local só empacota o código e dispara jobs.
+
+Uso:
+  PYTHONPATH=infra/oci .venv/bin/python infra/ds/jobs.py run --name probe --entrypoint infra/ds/probe.sh \
+      [--shape VM.Standard.E4.Flex --ocpus 2 --memory 16 | --shape VM.GPU.A10.1] [--private] \
+      [--env K=V ...] [--wait]
+  PYTHONPATH=infra/oci .venv/bin/python infra/ds/jobs.py logs <job_run_ocid>
+
+--private usa a sub-rede dmb-subnet-jobs (sem internet; só Service Gateway).
+Sem --private, o job usa a rede gerenciada do serviço (com internet), só para builds.
+"""
+from __future__ import annotations
+
+import argparse
+import io
+import subprocess
+import sys
+import time
+import zipfile
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+import oci
+
+from session import load_config
+import provision
+
+ROOT = Path(__file__).resolve().parents[2]
+EXCLUDE_DIRS = {".git", ".secrets", "models", "reports", "runs", "__pycache__", ".pytest_cache"}
+LOG_GROUP = "dmb-logs"
+
+
+def compartment(cfg_home) -> str:
+    return provision.find_compartment(oci.identity.IdentityClient(cfg_home), cfg_home["tenancy"])
+
+
+def code_zip() -> tuple[bytes, str]:
+    """Snapshot do repositório local (sem segredos, modelos e venvs)."""
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z:
+        z.writestr(zipfile.ZipInfo("dmb/"), "")
+        for p in sorted(ROOT.rglob("*")):
+            rel = p.relative_to(ROOT)
+            if any(part in EXCLUDE_DIRS or part.startswith(".venv") for part in rel.parts):
+                continue
+            if p.is_dir():
+                z.writestr(zipfile.ZipInfo(f"dmb/{rel.as_posix()}/"), "")
+                continue
+            info = zipfile.ZipInfo.from_file(p, f"dmb/{rel.as_posix()}")
+            info.compress_type = zipfile.ZIP_DEFLATED
+            z.writestr(info, p.read_bytes())
+    try:
+        rev = subprocess.run(["git", "-C", str(ROOT), "rev-parse", "--short", "HEAD"],
+                             capture_output=True, text=True).stdout.strip() or "sem-commit"
+    except OSError:
+        rev = "sem-commit"
+    return buf.getvalue(), rev
+
+
+def ensure_log_group(cfg, cid) -> str:
+    lc = oci.logging.LoggingManagementClient(cfg)
+    lg = next((g for g in lc.list_log_groups(cid, display_name=LOG_GROUP).data), None)
+    if lg is None:
+        lc.create_log_group(oci.logging.models.CreateLogGroupDetails(
+            compartment_id=cid, display_name=LOG_GROUP, description="Logs dos jobs do benchmark",
+            freeform_tags=provision.TAGS))
+        for _ in range(30):
+            lg = next((g for g in lc.list_log_groups(cid, display_name=LOG_GROUP).data), None)
+            if lg:
+                break
+            time.sleep(5)
+    return lg.id
+
+
+def subnet_id(cfg, cid) -> str:
+    vn = oci.core.VirtualNetworkClient(cfg)
+    return next(s.id for s in vn.list_subnets(cid, display_name="dmb-subnet-jobs").data
+                if s.lifecycle_state == "AVAILABLE")
+
+
+def run_job(name, entrypoint, shape, ocpus, memory, private, env, storage_gb=100, max_minutes=240, region=None):
+    cfg, home = load_config(region=region) if region else load_config(), load_config(region="us-ashburn-1")
+    cid = compartment(home)
+    ds = oci.data_science.DataScienceClient(cfg)
+    project = next(p.id for p in ds.list_projects(cid, lifecycle_state="ACTIVE").data
+                   if p.display_name == "benchmark-decisoes-ptbr")
+    log_group = ensure_log_group(cfg, cid)
+    flex = shape.endswith(".Flex")
+    infra = oci.data_science.models.StandaloneJobInfrastructureConfigurationDetails(
+        job_infrastructure_type="STANDALONE" if private else "ME_STANDALONE",
+        shape_name=shape, block_storage_size_in_gbs=storage_gb,
+        subnet_id=subnet_id(cfg, cid) if private else None,
+        job_shape_config_details=oci.data_science.models.JobShapeConfigDetails(
+            ocpus=ocpus, memory_in_gbs=memory) if flex else None)
+    payload, rev = code_zip()
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+    env = {"JOB_RUN_ENTRYPOINT": f"dmb/{entrypoint}", "DMB_CODE_REV": rev, **env}
+    if cfg["region"] == provision.GPU_REGION:
+        env.setdefault("DMB_BUCKET_ARTIFACTS", f"dmb-artefatos{provision.GPU_SUFFIX}")
+        env.setdefault("DMB_BUCKET_RESULTS", f"dmb-resultados{provision.GPU_SUFFIX}")
+        if not private:
+            env.setdefault("DMB_ISOLATION", "app")
+    job = ds.create_job(oci.data_science.models.CreateJobDetails(
+        project_id=project, compartment_id=cid, display_name=f"dmb-{name}-{stamp}",
+        freeform_tags=provision.TAGS,
+        job_configuration_details=oci.data_science.models.DefaultJobConfigurationDetails(
+            job_type="DEFAULT", environment_variables=env,
+            maximum_runtime_in_minutes=max_minutes),
+        job_infrastructure_configuration_details=infra,
+        job_log_configuration_details=oci.data_science.models.JobLogConfigurationDetails(
+            enable_logging=True, enable_auto_log_creation=True, log_group_id=log_group))).data
+    ds.create_job_artifact(job.id, payload, content_disposition="attachment; filename=dmb.zip")
+    run = ds.create_job_run(oci.data_science.models.CreateJobRunDetails(
+        project_id=project, compartment_id=cid, job_id=job.id,
+        display_name=f"dmb-{name}-{stamp}", freeform_tags=provision.TAGS)).data
+    print(f"job {job.id}\nrun {run.id}", flush=True)
+    return run.id
+
+
+def region_of(ocid: str) -> str:
+    return ocid.split(".")[3] if ocid.count(".") >= 4 and ocid.split(".")[3] else load_config()["region"]
+
+
+def wait(run_id, poll=30):
+    ds = oci.data_science.DataScienceClient(load_config(region=region_of(run_id)))
+    while True:
+        r = ds.get_job_run(run_id).data
+        if r.lifecycle_state in ("SUCCEEDED", "FAILED", "CANCELED", "DELETED"):
+            print("estado:", r.lifecycle_state, r.lifecycle_details or "", flush=True)
+            return r
+        time.sleep(poll)
+
+
+def logs(run_id, limit=500):
+    cfg = load_config(region=region_of(run_id))
+    ds = oci.data_science.DataScienceClient(cfg)
+    r = ds.get_job_run(run_id).data
+    if not r.log_details:
+        print("sem log configurado ainda")
+        return
+    search = oci.loggingsearch.LogSearchClient(cfg)
+    start = r.time_accepted - timedelta(minutes=5)
+    end = (r.time_finished or datetime.now(timezone.utc)) + timedelta(minutes=10)
+    query = (f'search "{r.compartment_id}/{r.log_details.log_group_id}/{r.log_details.log_id}" '
+             "| sort by datetime asc")
+    try:
+        res = search.search_logs(oci.loggingsearch.models.SearchLogsDetails(
+            time_start=start, time_end=end, search_query=query, is_return_field_info=False), limit=limit).data
+    except oci.exceptions.ServiceError as error:
+        print(f"logs indisponíveis ({error.status} {error.code}); log_id={r.log_details.log_id}")
+        return
+    for item in res.results or []:
+        print(item.data.get("logContent", {}).get("data", {}).get("message", ""))
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    r = sub.add_parser("run")
+    r.add_argument("--name", required=True)
+    r.add_argument("--entrypoint", required=True)
+    r.add_argument("--shape", default="VM.Standard.E4.Flex")
+    r.add_argument("--ocpus", type=float, default=2)
+    r.add_argument("--memory", type=float, default=16)
+    r.add_argument("--storage", type=int, default=100)
+    r.add_argument("--private", action="store_true")
+    r.add_argument("--env", nargs="*", default=[])
+    r.add_argument("--wait", action="store_true")
+    r.add_argument("--region", help="padrão: us-chicago-1; sa-saopaulo-1 para GPU")
+    lg = sub.add_parser("logs")
+    lg.add_argument("run_id")
+    args = ap.parse_args()
+    if args.cmd == "logs":
+        logs(args.run_id)
+        return
+    env = dict(kv.split("=", 1) for kv in args.env)
+    run_id = run_job(args.name, args.entrypoint, args.shape, args.ocpus, args.memory, args.private,
+                     env, storage_gb=args.storage, region=args.region)
+    if args.wait:
+        result = wait(run_id)
+        time.sleep(60)  # ingestão dos logs
+        logs(run_id)
+        sys.exit(0 if result.lifecycle_state == "SUCCEEDED" else 1)
+
+
+if __name__ == "__main__":
+    main()

infra/ds/probe.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Sonda do ambiente padrão dos jobs do Data Science.
+set -x
+whoami; echo "HOME=$HOME PWD=$PWD"; cat /etc/os-release | head -3; uname -m
+which python3 python conda curl tar unzip git 2>&1
+python3 --version 2>&1; python3 -c "import oci; print('oci', oci.__version__)" 2>&1
+df -h $HOME /tmp 2>&1 | tail -3; nproc; free -g | head -2
+nvidia-smi 2>&1 | head -5
+env | grep -E '^(JOB_|OCI_|CONDA|DMB_|NB_)' | sed -E 's/(TOKEN|KEY|SECRET)=.*/\1=***/' | sort
+ls "$PWD" | head; ls /home/datascience 2>&1 | head
+timeout 5 curl -sI https://pypi.org | head -1 || echo "sem internet"

infra/ds/results.py

@@ -0,0 +1,38 @@
+"""Resumo das execuções gravadas em dmb-resultados/runs/.
+
+Uso: PYTHONPATH=infra/oci .venv/bin/python infra/ds/results.py [filtro]
+"""
+import json
+import sys
+
+import oci
+
+from session import load_config
+
+BUCKET = "dmb-resultados"
+
+
+def main():
+    needle = sys.argv[1] if len(sys.argv) > 1 else ""
+    c = oci.object_storage.ObjectStorageClient(load_config())
+    ns = c.get_namespace().data
+    names = [o.name for o in oci.pagination.list_call_get_all_results(
+        c.list_objects, ns, BUCKET, prefix="runs/").data.objects]
+    for name in sorted(n for n in names if n.endswith("/run.json") and needle in n):
+        run = json.loads(c.get_object(ns, BUCKET, name).data.content)
+        line = {k: run.get(k) for k in ("run_id", "candidate", "device", "status")}
+        line["gate"] = run.get("egress_gate", {}).get("passed")
+        prefix = name.rsplit("/", 1)[0]
+        for n in names:
+            if n.startswith(prefix) and n.endswith(".json") and n != name:
+                rep = json.loads(c.get_object(ns, BUCKET, n).data.content)
+                if "summary" in rep:
+                    line |= rep["summary"] | {"load_s": rep.get("load_seconds")}
+                    lat = [r["latency_ms"] for r in rep.get("sanity", [])]
+                    if lat:
+                        line["latency_ms_median"] = sorted(lat)[len(lat) // 2]
+        print(json.dumps(line, ensure_ascii=False))
+
+
+if __name__ == "__main__":
+    main()

infra/ds/run_job.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+# Entrada dos jobs de benchmark: usa o Python base do job (SDK OCI + resource principal)
+# para o processo pai; a inferência roda no ambiente do candidato (ver harness/job.py).
+set -euo pipefail
+cd "$(dirname "$0")/../.."
+exec /opt/conda/bin/python3 -m harness.job

infra/oci/provision.py

@@ -1,8 +1,13 @@
-"""Provisionamento idempotente da fase F0 (tarefas 1, 2, 5, 6, 7, 8).
+"""Provisionamento idempotente da fase F0 (tarefas 1, 2, 5, 6, 7, 8) e do ambiente de execução.
+
+Ambiente de execução: a máquina local é só IDE e repositório. Os scripts rodam na VM
+dmb-vm (sub-rede de desenvolvimento, SSH só do IP autorizado) e em notebooks e jobs do
+OCI Data Science. A sub-rede privada dmb-subnet-jobs continua sem saída para a internet.
 
 Uso: PYTHONPATH=infra/oci .venv/bin/python infra/oci/provision.py
 Registra os artefatos criados em .secrets/artefatos-OCI.md.
 """
+import os
 import time
 from datetime import datetime, timezone
 
@@ -16,7 +21,19 @@ TAGS = {"projeto": "benchmark-decisoes-ptbr", "ambiente": "benchmark"}  # centro
 BUCKETS = ["entrada", "artefatos", "resultados", "logs-imutaveis"]
 VCN_CIDR = "10.20.0.0/16"
 SUBNET_CIDR = "10.20.1.0/24"
+DEV_SUBNET_CIDR = "10.20.2.0/24"
+SSH_ALLOWED_CIDR = os.environ.get("DMB_SSH_CIDR", "203.0.113.1/32")
+# VM de trabalho criada pelo usuário no console (o lançamento via API é negado neste compartment).
+VM_NAME = "instance-20261007-1206"
+VM_SSH_KEY = ".secrets/ssh/ssh-key-oci.key"
+VM_USER = "opc"
+NOTEBOOK_NAME = "dmb-notebook-cpu"
+NOTEBOOK_SHAPE = ("VM.Standard.E4.Flex", 4, 32)
 ARTIFACTS_FILE = CRED_DIR.parent / "artefatos-OCI.md"
+# Região secundária só para jobs de GPU (A10 sem capacidade em us-chicago-1).
+GPU_REGION = "sa-saopaulo-1"
+GPU_SUFFIX = "-gru"
+GPU_BUCKETS = ["artefatos", "resultados"]
 
 artifacts = []
 
@@ -64,21 +81,22 @@ def ensure_ds_project(cfg, cid):
         proj = ds.create_project(oci.data_science.models.CreateProjectDetails(
             compartment_id=cid, display_name=name, freeform_tags=TAGS,
             description="Benchmark de modelos de decisão PT-BR")).data
-    record("Data Science Project", name, proj.id, REGION)
+    record("Data Science Project", name, proj.id, cfg["region"])
+    return proj.id
 
 
-def ensure_buckets(cfg, cid):
+def ensure_buckets(cfg, cid, suffix="", buckets=BUCKETS):
     os_ = oci.object_storage.ObjectStorageClient(cfg)
     ns = os_.get_namespace().data
     existing = {b.name for b in oci.pagination.list_call_get_all_results(os_.list_buckets, ns, cid).data}
-    for short in BUCKETS:
-        name = f"dmb-{short}"
+    for short in buckets:
+        name = f"dmb-{short}{suffix}"
         if name not in existing:
             os_.create_bucket(ns, oci.object_storage.models.CreateBucketDetails(
                 name=name, compartment_id=cid, public_access_type="NoPublicAccess",
                 storage_tier="Standard", versioning="Enabled", freeform_tags=TAGS))
         b = os_.get_bucket(ns, name).data
-        record("Object Storage Bucket", f"{name} (namespace {ns})", b.id, REGION)
+        record("Object Storage Bucket", f"{name} (namespace {ns})", b.id, cfg["region"])
 
 
 def ensure_network(cfg, cid):
@@ -92,7 +110,7 @@ def ensure_network(cfg, cid):
             compartment_id=cid, display_name="dmb-vcn", cidr_blocks=[VCN_CIDR],
             dns_label="dmbvcn", freeform_tags=TAGS)).data
         vcn = comp(vn, vn.get_vcn(vcn.id), "lifecycle_state", "AVAILABLE").data
-    record("VCN", f"dmb-vcn ({VCN_CIDR})", vcn.id, REGION)
+    record("VCN", f"dmb-vcn ({VCN_CIDR})", vcn.id, cfg["region"])
 
     svc = next(s for s in vn.list_services().data if s.cidr_block.startswith("all-") and "services-in-oracle-services-network" in s.cidr_block)
     sgw = next((g for g in vn.list_service_gateways(cid, vcn_id=vcn.id).data
@@ -102,7 +120,7 @@ def ensure_network(cfg, cid):
             compartment_id=cid, vcn_id=vcn.id, display_name="dmb-sgw", freeform_tags=TAGS,
             services=[oci.core.models.ServiceIdRequestDetails(service_id=svc.id)])).data
         sgw = comp(vn, vn.get_service_gateway(sgw.id), "lifecycle_state", "AVAILABLE").data
-    record("Service Gateway", f"dmb-sgw ({svc.cidr_block})", sgw.id, REGION)
+    record("Service Gateway", f"dmb-sgw ({svc.cidr_block})", sgw.id, cfg["region"])
 
     rt = next((r for r in vn.list_route_tables(cid, vcn_id=vcn.id, display_name="dmb-rt-privada").data
                if r.lifecycle_state == "AVAILABLE"), None)
@@ -113,7 +131,7 @@ def ensure_network(cfg, cid):
                 destination=svc.cidr_block, destination_type="SERVICE_CIDR_BLOCK",
                 network_entity_id=sgw.id, description="Object Storage e serviços OCI via Service Gateway")])).data
         rt = comp(vn, vn.get_route_table(rt.id), "lifecycle_state", "AVAILABLE").data
-    record("Route Table", "dmb-rt-privada (somente Service Gateway)", rt.id, REGION)
+    record("Route Table", "dmb-rt-privada (somente Service Gateway)", rt.id, cfg["region"])
 
     sl = next((s for s in vn.list_security_lists(cid, vcn_id=vcn.id, display_name="dmb-sl-privada").data
                if s.lifecycle_state == "AVAILABLE"), None)
@@ -131,7 +149,7 @@ def ensure_network(cfg, cid):
                 oci.core.models.IngressSecurityRule(source=VCN_CIDR, protocol="all", description="Tráfego interno da VCN"),
             ])).data
         sl = comp(vn, vn.get_security_list(sl.id), "lifecycle_state", "AVAILABLE").data
-    record("Security List", "dmb-sl-privada (egresso só para serviços OCI e VCN)", sl.id, REGION)
+    record("Security List", "dmb-sl-privada (egresso só para serviços OCI e VCN)", sl.id, cfg["region"])
 
     sub = next((s for s in vn.list_subnets(cid, vcn_id=vcn.id, display_name="dmb-subnet-jobs").data
                 if s.lifecycle_state == "AVAILABLE"), None)
@@ -141,13 +159,14 @@ def ensure_network(cfg, cid):
             dns_label="jobs", prohibit_public_ip_on_vnic=True, route_table_id=rt.id,
             security_list_ids=[sl.id], freeform_tags=TAGS)).data
         sub = comp(vn, vn.get_subnet(sub.id), "lifecycle_state", "AVAILABLE").data
-    record("Subnet (privada)", f"dmb-subnet-jobs ({SUBNET_CIDR})", sub.id, REGION)
+    record("Subnet (privada)", f"dmb-subnet-jobs ({SUBNET_CIDR})", sub.id, cfg["region"])
 
     # O NAT é criado só por nat.py; aqui ele só é registrado, se existir.
     for nat in vn.list_nat_gateways(cid, vcn_id=vcn.id).data:
         if nat.lifecycle_state == "AVAILABLE":
             status = "bloqueado" if nat.block_traffic else "ATIVO"
-            record("NAT Gateway (temporário)", f"{nat.display_name} ({status})", nat.id, REGION)
+            record("NAT Gateway (temporário)", f"{nat.display_name} ({status})", nat.id, cfg["region"])
+    return vcn.id
 
 
 USER_FAMILIES = [
@@ -157,6 +176,9 @@ USER_FAMILIES = [
     "logging-family",
     "repos",
     "generative-ai-family",
+    "instance-family",
+    "volume-family",
+    "instance-agent-command-family",
 ]
 
 
@@ -192,18 +214,130 @@ def ensure_policies(idn, cid, user_id):
         f"Allow any-user to use log-content in {comp} where all {{{job}}}",
         f"Allow any-user to read repos in {comp} where all {{{job}}}",
         f"Allow service datascience to use virtual-network-family in {comp}",
+        # copy_object entre regiões (réplica para a região de GPU)
+        f"Allow service objectstorage-{REGION} to manage object-family in {comp}",
+        f"Allow service objectstorage-{GPU_REGION} to manage object-family in {comp}",
     ]
     ensure_policy(idn, cid, "dmb-jobs-menor-privilegio",
                   "Jobs de benchmark: leitura de entrada/artefatos e escrita de resultados/logs (tarefa 7)",
                   job_stmts)
 
 
+def ensure_workload_policy(idn, cid):
+    """VM (instance principal) e notebooks do Data Science (resource principal), sem dynamic groups."""
+    comp = f"compartment id {cid}"
+    who = (f"all {{request.principal.compartment.id = '{cid}', "
+           "any {request.principal.type = 'instance', request.principal.type = 'datasciencenotebooksession'}}")
+    stmts = [f"Allow any-user to {verb} in {comp} where {who}" for verb in (
+        "read buckets",
+        "manage objects",
+        "manage data-science-family",
+        "manage repos",
+        "use virtual-network-family",
+        "use log-groups",
+        "use log-content",
+        "use generative-ai-family",
+        "read compartments",
+    )]
+    ensure_policy(idn, cid, "dmb-workloads", "VM de trabalho e notebooks do benchmark", stmts)
+
+
+def ensure_dev_network(vn, cid, vcn_id):
+    cfg = {"region": REGION}  # a sub-rede de desenvolvimento só existe na região principal
+    igw = next((g for g in vn.list_internet_gateways(cid, vcn_id=vcn_id).data
+                if g.lifecycle_state == "AVAILABLE"), None)
+    if igw is None:
+        igw = vn.create_internet_gateway(oci.core.models.CreateInternetGatewayDetails(
+            compartment_id=cid, vcn_id=vcn_id, display_name="dmb-igw-dev", is_enabled=True,
+            freeform_tags=TAGS)).data
+        igw = oci.wait_until(vn, vn.get_internet_gateway(igw.id), "lifecycle_state", "AVAILABLE").data
+    record("Internet Gateway", "dmb-igw-dev (só para a sub-rede de desenvolvimento)", igw.id, cfg["region"])
+
+    rt = next((r for r in vn.list_route_tables(cid, vcn_id=vcn_id, display_name="dmb-rt-dev").data
+               if r.lifecycle_state == "AVAILABLE"), None)
+    if rt is None:
+        rt = vn.create_route_table(oci.core.models.CreateRouteTableDetails(
+            compartment_id=cid, vcn_id=vcn_id, display_name="dmb-rt-dev", freeform_tags=TAGS,
+            route_rules=[oci.core.models.RouteRule(destination="0.0.0.0/0", destination_type="CIDR_BLOCK",
+                                                   network_entity_id=igw.id)])).data
+        rt = oci.wait_until(vn, vn.get_route_table(rt.id), "lifecycle_state", "AVAILABLE").data
+    record("Route Table", "dmb-rt-dev (internet via IGW)", rt.id, cfg["region"])
+
+    ingress = [oci.core.models.IngressSecurityRule(
+        source=SSH_ALLOWED_CIDR, protocol="6", description="SSH só do IP autorizado",
+        tcp_options=oci.core.models.TcpOptions(destination_port_range=oci.core.models.PortRange(min=22, max=22)))]
+    sl = next((x for x in vn.list_security_lists(cid, vcn_id=vcn_id, display_name="dmb-sl-dev").data
+               if x.lifecycle_state == "AVAILABLE"), None)
+    if sl is None:
+        sl = vn.create_security_list(oci.core.models.CreateSecurityListDetails(
+            compartment_id=cid, vcn_id=vcn_id, display_name="dmb-sl-dev", freeform_tags=TAGS,
+            ingress_security_rules=ingress,
+            egress_security_rules=[oci.core.models.EgressSecurityRule(destination="0.0.0.0/0", protocol="all")])).data
+        sl = oci.wait_until(vn, vn.get_security_list(sl.id), "lifecycle_state", "AVAILABLE").data
+    elif [r.source for r in sl.ingress_security_rules] != [SSH_ALLOWED_CIDR]:
+        vn.update_security_list(sl.id, oci.core.models.UpdateSecurityListDetails(
+            ingress_security_rules=ingress, egress_security_rules=sl.egress_security_rules))
+    record("Security List", f"dmb-sl-dev (SSH de {SSH_ALLOWED_CIDR})", sl.id, cfg["region"])
+
+    sub = next((x for x in vn.list_subnets(cid, vcn_id=vcn_id, display_name="dmb-subnet-dev").data
+                if x.lifecycle_state == "AVAILABLE"), None)
+    if sub is None:
+        sub = vn.create_subnet(oci.core.models.CreateSubnetDetails(
+            compartment_id=cid, vcn_id=vcn_id, display_name="dmb-subnet-dev", cidr_block=DEV_SUBNET_CIDR,
+            dns_label="dev", prohibit_public_ip_on_vnic=False, route_table_id=rt.id,
+            security_list_ids=[sl.id], freeform_tags=TAGS)).data
+        sub = oci.wait_until(vn, vn.get_subnet(sub.id), "lifecycle_state", "AVAILABLE").data
+    record("Subnet (desenvolvimento)", f"dmb-subnet-dev ({DEV_SUBNET_CIDR})", sub.id, cfg["region"])
+    return sub.id
+
+
+def ensure_vm(cfg, cid):
+    """Registra a VM de trabalho, os volumes anexados e o acesso SSH. Não cria instâncias."""
+    cc = oci.core.ComputeClient(cfg)
+    vm = next((i for i in cc.list_instances(cid, display_name=VM_NAME).data
+               if i.lifecycle_state not in ("TERMINATED", "TERMINATING")), None)
+    if vm is None:
+        print(f"aviso: VM {VM_NAME} não encontrada")
+        return None
+    vn = oci.core.VirtualNetworkClient(cfg)
+    bs = oci.core.BlockstorageClient(cfg)
+    vnic = vn.get_vnic(cc.list_vnic_attachments(cid, instance_id=vm.id).data[0].vnic_id).data
+    record("Compute Instance", f"{VM_NAME} ({vm.shape}, {vm.shape_config.ocpus:.0f} OCPU, "
+           f"{vm.shape_config.memory_in_gbs:.0f} GB, Oracle Linux 9, IP público {vnic.public_ip}, "
+           f"usuário {VM_USER}, chave {VM_SSH_KEY})", vm.id, cfg["region"])
+    for att in cc.list_boot_volume_attachments(vm.availability_domain, cid, instance_id=vm.id).data:
+        bv = bs.get_boot_volume(att.boot_volume_id).data
+        record("Boot Volume", f"{bv.display_name} ({bv.size_in_gbs} GB, raiz /)", bv.id, cfg["region"])
+    for att in cc.list_volume_attachments(cid, instance_id=vm.id).data:
+        if att.lifecycle_state == "ATTACHED":
+            v = bs.get_volume(att.volume_id).data
+            record("Block Volume", f"{v.display_name} ({v.size_in_gbs} GB, {att.attachment_type}, "
+                   f"{att.device}, montado em /data)", v.id, cfg["region"])
+    return vnic.public_ip
+
+
+def ensure_notebook(cfg, cid, project_id):
+    ds = oci.data_science.DataScienceClient(cfg)
+    nb = next((n for n in ds.list_notebook_sessions(cid, project_id=project_id).data
+               if n.display_name == NOTEBOOK_NAME and n.lifecycle_state not in ("DELETED", "DELETING")), None)
+    if nb is None:
+        shape, ocpus, mem = NOTEBOOK_SHAPE
+        nb = ds.create_notebook_session(oci.data_science.models.CreateNotebookSessionDetails(
+            compartment_id=cid, project_id=project_id, display_name=NOTEBOOK_NAME, freeform_tags=TAGS,
+            notebook_session_config_details=oci.data_science.models.NotebookSessionConfigDetails(
+                shape=shape, block_storage_size_in_gbs=200,
+                notebook_session_shape_config_details=oci.data_science.models.NotebookSessionShapeConfigDetails(
+                    ocpus=ocpus, memory_in_gbs=mem)))).data
+    record("Data Science Notebook Session", f"{NOTEBOOK_NAME} ({NOTEBOOK_SHAPE[0]}, rede gerenciada, "
+           f"estado {nb.lifecycle_state})", nb.id, cfg["region"])
+
+
 def write_artifacts():
     now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
     lines = [
         "# Artefatos OCI — benchmark de modelos de decisão",
         "",
-        f"Atualizado em {now} por `infra/oci/provision.py`. Região dos recursos: `{REGION}`.",
+        f"Atualizado em {now} por `infra/oci/provision.py`. Região principal: `{REGION}`; GPU também em `{GPU_REGION}`.",
         f"Tags livres aplicadas: {', '.join(f'`{k}={v}`' for k, v in TAGS.items())} (`centro_custo` pendente, Q13).",
         "",
         "| Tipo | Nome | OCID | Região |",
@@ -219,9 +353,24 @@ def main():
     idn = oci.identity.IdentityClient(home)
     cid = find_compartment(idn, home["tenancy"])
     ensure_policies(idn, cid, home["user"])
-    ensure_ds_project(cfg, cid)
+    ensure_workload_policy(idn, cid)
+    project_id = ensure_ds_project(cfg, cid)
     ensure_buckets(cfg, cid)
-    ensure_network(cfg, cid)
+    vcn_id = ensure_network(cfg, cid)
+    ensure_dev_network(oci.core.VirtualNetworkClient(cfg), cid, vcn_id)
+    ensure_vm(cfg, cid)
+    ensure_notebook(cfg, cid, project_id)
+
+    gru = load_config(region=GPU_REGION)
+    ensure_ds_project(gru, cid)
+    ensure_buckets(gru, cid, suffix=GPU_SUFFIX, buckets=GPU_BUCKETS)
+    try:
+        ensure_network(gru, cid)
+    except oci.exceptions.ServiceError as error:
+        if error.code != "LimitExceeded":
+            raise
+        # Sem VCN disponível na região: jobs de GPU usam rede gerenciada + isolamento no processo.
+        print(f"aviso: VCN indisponível em {GPU_REGION} ({error.message[:80]})")
     write_artifacts()
     for a in artifacts:
         print(" | ".join(a))

infra/oci/replicate.py

@@ -0,0 +1,63 @@
+"""Réplica server-side dos artefatos para a região de GPU (copy_object entre regiões).
+
+Copia os pesos (models/) e o ambiente atual de cada candidato (envs/<c>/latest.json e o
+pacote que ele referencia). Objetos já presentes com o mesmo tamanho são pulados.
+
+Uso: PYTHONPATH=infra/oci .venv/bin/python infra/oci/replicate.py [--wait]
+"""
+import json
+import sys
+import time
+
+import oci
+
+import provision
+from session import load_config
+
+SRC_BUCKET = "dmb-artefatos"
+DST_BUCKET = f"dmb-artefatos{provision.GPU_SUFFIX}"
+
+
+def main():
+    src = oci.object_storage.ObjectStorageClient(load_config())
+    dst = oci.object_storage.ObjectStorageClient(load_config(region=provision.GPU_REGION))
+    ns = src.get_namespace().data
+    objects = oci.pagination.list_call_get_all_results(
+        src.list_objects, ns, SRC_BUCKET, fields="size").data.objects
+    sizes = {o.name: o.size for o in objects}
+    wanted = [n for n in sizes if n.startswith("models/")]
+    for cand in ("laya", "gliner", "semif", "rizzo_flow"):
+        latest = f"envs/{cand}/latest.json"
+        if latest in sizes:
+            obj = json.loads(src.get_object(ns, SRC_BUCKET, latest).data.content)["object"]
+            wanted += [latest, obj, obj.replace(".tar.gz", "-env-manifest.json")]
+    cuda = "runtimes/rizzo_flow/cuda-latest.json"
+    if cuda in sizes:
+        wanted += [cuda, json.loads(src.get_object(ns, SRC_BUCKET, cuda).data.content)["object"]]
+    existing = {o.name: o.size for o in oci.pagination.list_call_get_all_results(
+        dst.list_objects, ns, DST_BUCKET, fields="size").data.objects}
+    requests = []
+    for name in wanted:
+        if name not in sizes:
+            continue
+        if existing.get(name) == sizes[name] and not name.endswith("latest.json"):
+            continue
+        r = src.copy_object(ns, SRC_BUCKET, oci.object_storage.models.CopyObjectDetails(
+            source_object_name=name, destination_region=provision.GPU_REGION,
+            destination_namespace=ns, destination_bucket=DST_BUCKET, destination_object_name=name))
+        requests.append((name, r.headers["opc-work-request-id"]))
+    total = sum(sizes[n] for n, _ in requests)
+    print(f"{len(requests)} cópias iniciadas ({total / 1e9:.2f} GB) para {DST_BUCKET} em {provision.GPU_REGION}")
+    if "--wait" in sys.argv:
+        pending = dict(requests)
+        while pending:
+            for name, wr in list(pending.items()):
+                status = src.get_work_request(wr).data.status
+                if status in ("COMPLETED", "FAILED", "CANCELED"):
+                    print(status, name, flush=True)
+                    pending.pop(name)
+            time.sleep(20)
+
+
+if __name__ == "__main__":
+    main()

infra/vm/bootstrap.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Preparação idempotente da VM de trabalho (Oracle Linux 9). Rodar na VM:
+#   bash infra/vm/bootstrap.sh
+set -euo pipefail
+sudo dnf install -y -q git rsync tmux jq python3.12 python3.12-pip python3.12-devel gcc gcc-c++ >/dev/null
+[ -d "$HOME/.venv-oci" ] || python3.12 -m venv "$HOME/.venv-oci"
+"$HOME/.venv-oci/bin/pip" install -q --upgrade pip
+"$HOME/.venv-oci/bin/pip" install -q "oci==2.187.2" oci-cli "huggingface_hub==0.36.2" jsonschema fastapi uvicorn pyyaml pytest httpx
+grep -q OCI_CLI_AUTH "$HOME/.bashrc" || cat >> "$HOME/.bashrc" <<'RC'
+export OCI_CLI_AUTH=instance_principal
+export DMB_OCI_AUTH=instance_principal
+export PATH=$HOME/.venv-oci/bin:$PATH
+RC
+echo "bootstrap ok: $(python3.12 --version), oci $("$HOME/.venv-oci/bin/python" -c 'import oci;print(oci.__version__)')"

infra/vm/remote.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Máquina local = IDE + repositório; execução na VM dmb-vm.
+#   infra/vm/remote.sh sync              # envia o repositório (sem .secrets, models, venvs)
+#   infra/vm/remote.sh run <comando...>  # sincroniza e roda o comando na VM, dentro do repositório
+#   infra/vm/remote.sh ssh               # abre um shell na VM
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+KEY="$ROOT/.secrets/ssh/ssh-key-oci.key"
+IP="$(grep -oE 'IP público [0-9.]+' "$ROOT/.secrets/artefatos-OCI.md" | head -1 | awk '{print $3}')"
+[ -n "$IP" ] || { echo "IP da VM não encontrado em .secrets/artefatos-OCI.md" >&2; exit 1; }
+SSH=(ssh -i "$KEY" -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 "opc@$IP")
+REMOTE_DIR=/data/repo
+
+sync() {
+  rsync -az --delete -e "ssh -i $KEY -o StrictHostKeyChecking=accept-new" \
+    --exclude '.git/' --exclude '.secrets/' --exclude 'models/' --exclude '.venv*/' \
+    --exclude '__pycache__/' --exclude 'reports/sanity/' --exclude 'runs/' \
+    "$ROOT/" "opc@$IP:$REMOTE_DIR/"
+}
+
+case "${1:-}" in
+  sync) sync ;;
+  run) shift; sync; "${SSH[@]}" "cd $REMOTE_DIR && bash -lc $(printf '%q' "$*")" ;;
+  ssh) "${SSH[@]}" ;;
+  *) sed -n '2,6p' "$0"; exit 1 ;;
+esac