setup models

harness/candidates.py

@@ -0,0 +1,33 @@
+"""Construção dos adaptadores a partir do manifesto de artefatos."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from adapters import adapter_class
+
+ROOT = Path(__file__).resolve().parents[1]
+MANIFEST = ROOT / "env" / "artifacts-manifest.json"
+MODEL_REPO = {
+    "laya": "convaiinnovations/laya-multilingual",
+    "semif": "Qwen/Qwen3.5-4B",
+    "gliner": "fastino/GLiNER2.5-multi-Decide",
+    "rizzo_flow": "rizzoaiacademy/rizzo-flow",
+}
+
+
+def model_location(candidate: str, models_root: Path) -> tuple[Path, str]:
+    """Diretório local e revisão fixada do candidato, segundo o manifesto."""
+    manifest = json.loads(MANIFEST.read_text())
+    repo = MODEL_REPO[candidate]
+    entry = next(a for a in manifest["artifacts"]
+                 if a["candidate"] == candidate and a["kind"] == "hf_model" and a["origin"] == f"hf://{repo}")
+    rev = entry["revision"]
+    return models_root / candidate / repo.replace("/", "__") / rev, rev
+
+
+def build_adapter(candidate: str, models_root: str | Path, device: str = "auto",
+                  context_limit: int | None = None, **extra):
+    model_dir, revision = model_location(candidate, Path(models_root))
+    return adapter_class(candidate)(model_dir, revision, device=device,
+                                    context_limit=context_limit, **extra)

harness/image_manifest.py

@@ -0,0 +1,29 @@
+"""Imprime o manifesto de versões do ambiente atual (tarefa 12)."""
+import json
+import os
+import platform
+import sys
+from importlib import metadata
+
+
+def collect() -> dict:
+    packages = sorted((d.metadata["Name"], d.version) for d in metadata.distributions())
+    info = {
+        "candidate": os.environ.get("DMB_CANDIDATE"),
+        "python": sys.version.split()[0],
+        "platform": platform.platform(),
+        "base_image": os.environ.get("DMB_BASE_IMAGE"),
+        "packages": {name: version for name, version in packages},
+    }
+    try:
+        import torch
+        info["torch_cuda"] = torch.version.cuda
+        info["cudnn"] = torch.backends.cudnn.version()
+    except ImportError:
+        pass
+    return info
+
+
+if __name__ == "__main__":
+    json.dump(collect(), sys.stdout, indent=2)
+    print()

harness/job.py

@@ -0,0 +1,193 @@
+"""Ponto de entrada dos jobs do OCI Data Science (tarefas 10, 17–19).
+
+O processo pai fala só com o Object Storage (via Service Gateway): baixa os artefatos do
+candidato, confere o SHA-256 contra o manifesto, roda o gate de egresso e, no fim, envia
+resultados e logs. A inferência roda num processo filho com o bloqueio de rede ativo.
+
+Variáveis: DMB_CANDIDATE, DMB_MODE (gate|sanity), DMB_DEVICE (auto|cpu|cuda), DMB_RUN_ID,
+DMB_MODELS_ROOT. O ambiente Python do candidato vem de dmb-artefatos/envs/<candidato>/latest.json,
+publicado pelo job de build (infra/ds/build_env.sh).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import socket
+import subprocess
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+MANIFEST = ROOT / "env" / "artifacts-manifest.json"
+BUCKET_ARTIFACTS = os.environ.get("DMB_BUCKET_ARTIFACTS", "dmb-artefatos")
+BUCKET_RESULTS = os.environ.get("DMB_BUCKET_RESULTS", "dmb-resultados")
+EGRESS_PROBES = [("huggingface.co", 443), ("pypi.org", 443), ("github.com", 443)]
+
+
+def oci_clients():
+    import oci
+
+    if os.environ.get("OCI_RESOURCE_PRINCIPAL_VERSION"):
+        signer = oci.auth.signers.get_resource_principals_signer()
+        client = oci.object_storage.ObjectStorageClient({}, signer=signer)
+    else:
+        sys.path.insert(0, str(ROOT / "infra" / "oci"))
+        from session import load_config
+        client = oci.object_storage.ObjectStorageClient(load_config())
+    return client, client.get_namespace().data
+
+
+def sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as f:
+        for chunk in iter(lambda: f.read(1 << 22), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def fetch_artifacts(candidate: str, models_root: Path) -> dict:
+    """Tarefa 18: só artefatos do manifesto, com hash conferido."""
+    client, ns = oci_clients()
+    manifest = json.loads(MANIFEST.read_text())
+    entries = [a for a in manifest["artifacts"] if a["candidate"] == candidate and a["kind"] == "hf_model"]
+    started, total = time.perf_counter(), 0
+    for a in entries:
+        target = models_root / a["object"].removeprefix("models/")
+        if not (target.is_file() and target.stat().st_size == a["bytes"] and sha256(target) == a["sha256"]):
+            target.parent.mkdir(parents=True, exist_ok=True)
+            obj = client.get_object(ns, BUCKET_ARTIFACTS, a["object"])
+            with target.open("wb") as f:
+                for chunk in obj.data.raw.stream(1 << 22, decode_content=False):
+                    f.write(chunk)
+            if sha256(target) != a["sha256"]:
+                raise SystemExit(f"SHA-256 divergente: {a['object']}")
+        total += a["bytes"]
+    return {"files": len(entries), "bytes": total, "seconds": round(time.perf_counter() - started, 1)}
+
+
+def install_env(candidate: str) -> tuple[Path, dict]:
+    """Baixa o ambiente publicado pelo build, confere o SHA-256 e extrai em $HOME/envs."""
+    import tarfile
+
+    from harness import objstore
+
+    home = Path(os.environ.get("HOME", "/home/datascience"))
+    latest = json.loads(objstore.get(BUCKET_ARTIFACTS, f"envs/{candidate}/latest.json",
+                                     Path("/tmp/latest.json")).read_text())
+    if Path(latest["home"]) != home:
+        raise SystemExit(f"ambiente construído para HOME={latest['home']}, job em {home}")
+    tarball = objstore.get(BUCKET_ARTIFACTS, latest["object"], Path("/tmp/env.tar.gz"))
+    if sha256(tarball) != latest["sha256"]:
+        raise SystemExit("SHA-256 do ambiente divergente")
+    with tarfile.open(tarball) as tar:
+        tar.extractall(home)
+    tarball.unlink()
+    return home / "envs" / candidate / "bin" / "python", latest
+
+
+def install_rizzo_cuda_runtime() -> Path:
+    """llama.cpp CUDA compilado para a imagem dos jobs (infra/ds/build_llama_cuda.sh)."""
+    import tarfile
+
+    from harness import objstore
+
+    home = Path(os.environ.get("HOME", "/home/datascience"))
+    latest = json.loads(objstore.get(BUCKET_ARTIFACTS, "runtimes/rizzo_flow/cuda-latest.json",
+                                     Path("/tmp/cuda-latest.json")).read_text())
+    tarball = objstore.get(BUCKET_ARTIFACTS, latest["object"], Path("/tmp/llama-cuda.tar.gz"))
+    if sha256(tarball) != latest["sha256"]:
+        raise SystemExit("SHA-256 do runtime CUDA divergente")
+    with tarfile.open(tarball) as tar:
+        tar.extractall(home)
+    return home / latest["dir"]
+
+
+def has_gpu() -> bool:
+    return subprocess.run(["sh", "-c", "command -v nvidia-smi && nvidia-smi -L"],
+                          capture_output=True).returncode == 0
+
+
+def egress_gate() -> dict:
+    """Tarefas 17 e 19: o gate passa só se nenhuma saída para a internet funcionar."""
+    probes = []
+    for host, port in EGRESS_PROBES:
+        try:
+            with socket.create_connection((host, port), timeout=5):
+                outcome = "CONECTOU"
+        except OSError as error:
+            outcome = f"bloqueado ({type(error).__name__})"
+        probes.append({"host": host, "port": port, "outcome": outcome})
+    return {"passed": all(p["outcome"] != "CONECTOU" for p in probes), "probes": probes}
+
+
+def upload_dir(local: Path, bucket: str, prefix: str) -> list[str]:
+    client, ns = oci_clients()
+    sent = []
+    for f in sorted(p for p in local.rglob("*") if p.is_file()):
+        name = f"{prefix}/{f.relative_to(local).as_posix()}"
+        with f.open("rb") as body:
+            client.put_object(ns, bucket, name, body)
+        sent.append(name)
+    return sent
+
+
+def main():
+    candidate = os.environ["DMB_CANDIDATE"]
+    mode = os.environ.get("DMB_MODE", "sanity")
+    device = os.environ.get("DMB_DEVICE", "auto")
+    run_id = os.environ.setdefault(
+        "DMB_RUN_ID", f"{datetime.now(timezone.utc):%Y%m%dT%H%M%SZ}-{candidate}-{mode}")
+    models_root = Path(os.environ.get("DMB_MODELS_ROOT", Path.home() / "models"))
+    if device == "auto":
+        device = "cuda" if has_gpu() else "cpu"
+    out = Path("/tmp/dmb-out") / run_id
+    out.mkdir(parents=True, exist_ok=True)
+
+    run = {"run_id": run_id, "candidate": candidate, "mode": mode, "device": device,
+           "started_at": datetime.now(timezone.utc).isoformat(timespec="seconds")}
+    image_manifest = ROOT / "image-manifest.json"
+    if image_manifest.is_file():
+        run["image"] = json.loads(image_manifest.read_text())
+
+    run["egress_gate"] = egress_gate()
+    # DMB_ISOLATION=app: só onde não há sub-rede privada (região de GPU sem VCN disponível).
+    # A rede tem saída; o isolamento fica a cargo do bloqueio no processo (dmb.offline + self_test).
+    run["isolation"] = os.environ.get("DMB_ISOLATION", "network")
+    if not run["egress_gate"]["passed"] and mode != "gate" and run["isolation"] != "app":
+        run["status"] = "abortado: gate de egresso reprovado"
+    else:
+        if mode != "gate":
+            run["artifacts"] = fetch_artifacts(candidate, models_root)
+            python, run["env"] = install_env(candidate)
+            child_env = dict(os.environ)
+            if candidate == "rizzo_flow":
+                # CPU: binário oficial; CUDA: build próprio (o oficial exige GLIBC 2.38).
+                child_env["RIZZO_LLAMA_DIR"] = str(
+                    install_rizzo_cuda_runtime() if device == "cuda"
+                    else python.parents[1] / "runtimes" / "llama-b11081-linux-x64-cpu")
+                run["rizzo_runtime"] = child_env["RIZZO_LLAMA_DIR"]
+            cmd = [str(python), "-m", f"harness.{mode}", "--candidate", candidate,
+                   "--models-root", str(models_root), "--device", device, "--out", str(out)]
+            proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, env=child_env)
+            print(proc.stdout[-4000:])
+            print(proc.stderr[-4000:], file=sys.stderr)
+            (out / "stdout.log").write_text(proc.stdout)
+            (out / "stderr.log").write_text(proc.stderr)
+            run["exit_code"] = proc.returncode
+            run["status"] = "ok" if proc.returncode == 0 else "falhou"
+        else:
+            run["status"] = "ok" if run["egress_gate"]["passed"] else "gate reprovado"
+
+    run["finished_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds")
+    (out / "run.json").write_text(json.dumps(run, indent=2, ensure_ascii=False) + "\n")
+    print(json.dumps({k: run[k] for k in ("run_id", "status", "egress_gate")}, ensure_ascii=False))
+    sent = upload_dir(out, BUCKET_RESULTS, f"runs/{run_id}")
+    print(f"{len(sent)} arquivos enviados para {BUCKET_RESULTS}/runs/{run_id}")
+    sys.exit(0 if run["status"] == "ok" else 1)
+
+
+if __name__ == "__main__":
+    main()

harness/objstore.py

@@ -0,0 +1,67 @@
+"""Acesso mínimo ao Object Storage, usável com o Python base dos jobs (resource principal).
+
+  python -m harness.objstore get <bucket> <objeto> <destino>
+  python -m harness.objstore put <bucket> <objeto> <origem> [--sha256]
+"""
+from __future__ import annotations
+
+import hashlib
+import os
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def client():
+    import oci
+
+    if os.environ.get("OCI_RESOURCE_PRINCIPAL_VERSION"):
+        c = oci.object_storage.ObjectStorageClient({}, signer=oci.auth.signers.get_resource_principals_signer())
+    elif os.environ.get("DMB_OCI_AUTH") == "instance_principal":
+        signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
+        c = oci.object_storage.ObjectStorageClient({"region": signer.region}, signer=signer)
+    else:
+        sys.path.insert(0, str(ROOT / "infra" / "oci"))
+        from session import load_config
+        c = oci.object_storage.ObjectStorageClient(load_config())
+    return c, c.get_namespace().data
+
+
+def sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with Path(path).open("rb") as f:
+        for chunk in iter(lambda: f.read(1 << 22), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def get(bucket: str, name: str, dest: Path) -> Path:
+    c, ns = client()
+    dest = Path(dest)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    obj = c.get_object(ns, bucket, name)
+    with dest.open("wb") as f:
+        for chunk in obj.data.raw.stream(1 << 22, decode_content=False):
+            f.write(chunk)
+    return dest
+
+
+def put(bucket: str, name: str, src: Path, metadata: dict | None = None) -> None:
+    from oci.object_storage import UploadManager
+
+    c, ns = client()
+    UploadManager(c, allow_parallel_uploads=True, parallel_process_count=8).upload_file(
+        ns, bucket, name, str(src), metadata=metadata or {})
+
+
+if __name__ == "__main__":
+    cmd, bucket, name, path = sys.argv[1:5]
+    if cmd == "get":
+        get(bucket, name, Path(path))
+    elif cmd == "put":
+        meta = {"sha256": sha256(Path(path))} if "--sha256" in sys.argv else {}
+        put(bucket, name, Path(path), meta)
+        print(meta.get("sha256", ""))
+    else:
+        sys.exit(__doc__)

harness/sanity.py

@@ -0,0 +1,99 @@
+"""Conformidade e sanidade de um candidato real (tarefas 41, 49–51).
+
+Executa as fixtures de conformidade e os casos de sanidade (decisões triviais, os três
+tipos de pergunta e a mesma pergunta com as opções em ordem invertida) e grava um
+relatório JSON. Falha (código 1) só por violação de contrato; os acertos de sanidade
+são reportados, não exigidos.
+
+Uso: python -m harness.sanity --candidate laya --models-root models --device cpu --out reports/sanity
+"""
+import argparse
+import json
+import platform
+import sys
+import time
+from pathlib import Path
+
+from dmb.offline import enforce_offline
+
+FIXTURES = Path(__file__).resolve().parents[1] / "contracts" / "jev" / "fixtures"
+
+
+def predicted(answer: dict) -> str:
+    if answer["type"] == "noul":
+        return "true" if answer["noul"] >= 0.5 else "false"
+    return max(answer["probabilities"], key=answer["probabilities"].get)
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--candidate", required=True)
+    ap.add_argument("--models-root", default="models")
+    ap.add_argument("--device", default="auto")
+    ap.add_argument("--out", default="reports/sanity")
+    args = ap.parse_args()
+
+    enforce_offline()
+    from dmb.offline import self_test
+    guard = self_test()
+    if not guard["process_guard"].startswith("ok"):
+        sys.exit(f"bloqueio de rede inativo: {guard}")
+    from dmb.jev import JevError
+    from dmb.server import systemone
+    from harness.candidates import build_adapter
+
+    adapter = build_adapter(args.candidate, args.models_root, args.device)
+    started = time.perf_counter()
+    adapter.load()
+    load_s = time.perf_counter() - started
+
+    violations, conformance = [], []
+    for case in json.loads((FIXTURES / "conformance.json").read_text())["cases"]:
+        try:
+            systemone(adapter, case["request"])
+            outcome = "ok"
+        except JevError as error:
+            outcome = error.error_type
+        conformance.append({"name": case["name"], "expected": case["expect"], "got": outcome})
+        if outcome != case["expect"]:
+            violations.append(f"conformance:{case['name']}: esperado {case['expect']}, obtido {outcome}")
+
+    sanity = []
+    for case in json.loads((FIXTURES / "sanity.json").read_text())["cases"]:
+        q = case["question"]
+        request = {"model": adapter.model_id, "state": case["state"], "questions": {"q": q}}
+        r = systemone(adapter, request)
+        row = {"name": case["name"], "gold": case["gold"], "pred": predicted(r["answers"]["q"]),
+               "latency_ms": round(r["metadata"]["latency_ms"], 1), "answer": r["answers"]["q"]}
+        if q["type"] == "choice":
+            flipped = dict(reversed(list(q["criteria"].items())))
+            r2 = systemone(adapter, {**request, "questions": {"q": {**q, "criteria": flipped}}})
+            if list(r2["answers"]["q"]["probabilities"]) != list(flipped):
+                violations.append(f"sanity:{case['name']}: ordem das opções não preservada")
+            row["pred_permuted"] = predicted(r2["answers"]["q"])
+            row["permutation_invariant"] = row["pred_permuted"] == row["pred"]
+        row["correct"] = row["pred"] == row["gold"]
+        sanity.append(row)
+
+    report = {
+        "candidate": adapter.describe(), "device": args.device, "host": platform.node(),
+        "load_seconds": round(load_s, 2), **guard,
+        "conformance": conformance, "sanity": sanity,
+        "summary": {
+            "contract_violations": violations,
+            "sanity_correct": f"{sum(r['correct'] for r in sanity)}/{len(sanity)}",
+            "permutation_invariant": f"{sum(r.get('permutation_invariant', False) for r in sanity)}"
+                                     f"/{sum('permutation_invariant' in r for r in sanity)}",
+        },
+    }
+    out = Path(args.out)
+    out.mkdir(parents=True, exist_ok=True)
+    path = out / f"{args.candidate}-{args.device}.json"
+    path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n")
+    print(json.dumps(report["summary"], ensure_ascii=False), "load_s=", report["load_seconds"])
+    print("relatório:", path)
+    sys.exit(1 if violations else 0)
+
+
+if __name__ == "__main__":
+    main()

harness/serve.py

@@ -0,0 +1,31 @@
+"""Sobe um candidato em POST /v1/systemone, sem acesso à rede externa.
+
+Uso: python -m harness.serve --candidate laya --models-root models [--device cpu] [--port 8080]
+"""
+import argparse
+
+from dmb.offline import enforce_offline
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--candidate", required=True)
+    ap.add_argument("--models-root", default="models")
+    ap.add_argument("--device", default="auto")
+    ap.add_argument("--context-limit", type=int)
+    ap.add_argument("--host", default="127.0.0.1")
+    ap.add_argument("--port", type=int, default=8080)
+    args = ap.parse_args()
+
+    enforce_offline()
+    import uvicorn
+
+    from dmb.server import create_app
+    from harness.candidates import build_adapter
+
+    adapter = build_adapter(args.candidate, args.models_root, args.device, args.context_limit)
+    uvicorn.run(create_app(adapter), host=args.host, port=args.port, log_level="warning")
+
+
+if __name__ == "__main__":
+    main()

harness/verify_bucket.py

@@ -0,0 +1,31 @@
+"""Audita o bucket de artefatos: o conteúdo real de cada objeto bate com o SHA-256 do manifesto?
+
+Uso (na VM ou num job): python -m harness.verify_bucket
+"""
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+from harness.objstore import client
+
+MANIFEST = Path(__file__).resolve().parents[1] / "env" / "artifacts-manifest.json"
+
+
+def main():
+    c, ns = client()
+    manifest = json.loads(MANIFEST.read_text())
+    bad = 0
+    for a in manifest["artifacts"]:
+        h = hashlib.sha256()
+        for chunk in c.get_object(ns, manifest["bucket"], a["object"]).data.raw.stream(1 << 22, decode_content=False):
+            h.update(chunk)
+        ok = h.hexdigest() == a["sha256"]
+        bad += not ok
+        print(("ok  " if ok else "DIVERGENTE  ") + a["object"], flush=True)
+    print(f"{len(manifest['artifacts']) - bad}/{len(manifest['artifacts'])} artefatos conferem")
+    sys.exit(1 if bad else 0)
+
+
+if __name__ == "__main__":
+    main()