setup models

dmb/__init__.py

@@ -0,0 +1 @@
+"""Benchmark de modelos de decisão PT-BR: código comum."""

dmb/adapter.py

@@ -0,0 +1,50 @@
+"""Interface comum dos adaptadores JEV locais (tarefas 42–47)."""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+
+class Adapter(ABC):
+    """Um candidato servido atrás de POST /v1/systemone.
+
+    `decide` recebe o estado já serializado (texto) e as perguntas no formato JEV e
+    devolve as respostas JEV de cada pergunta, mais o total de tokens de entrada.
+    """
+
+    name: str
+    adapter_version = "0.1.0"
+    probabilities_source = "native"
+
+    def __init__(self, model_dir: str | Path, revision: str, device: str = "auto",
+                 context_limit: int | None = None):
+        self.model_dir = Path(model_dir)
+        self.revision = revision
+        self.device = device
+        self._context_limit = context_limit
+
+    @property
+    def model_id(self) -> str:
+        return f"{self.name}@{self.revision[:12]}"
+
+    @property
+    def context_limit(self) -> int:
+        return self._context_limit or self.native_context_limit()
+
+    @abstractmethod
+    def load(self) -> None: ...
+
+    @abstractmethod
+    def native_context_limit(self) -> int: ...
+
+    @abstractmethod
+    def count_tokens(self, text: str) -> int: ...
+
+    @abstractmethod
+    def decide(self, state: str, questions: dict) -> tuple[dict, int]: ...
+
+    def describe(self) -> dict:
+        return {"name": self.name, "model_id": self.model_id, "revision": self.revision,
+                "adapter_version": self.adapter_version, "device": self.device,
+                "context_limit": self.context_limit,
+                "probabilities_source": self.probabilities_source}

dmb/canonical.py

@@ -0,0 +1,76 @@
+"""Formato canônico, serialização do estado e conversão para JEV (tarefas 36, 53 e 54)."""
+from __future__ import annotations
+
+from typing import Callable, Iterator
+
+from .jev import validate_request
+
+SERIALIZATION_VERSION = "state-text-v1"
+
+
+def iter_questions(example: dict) -> Iterator[dict]:
+    """Um exemplo traz uma pergunta (campos na raiz) ou várias (`questions`, tarefa 65)."""
+    if "questions" in example:
+        yield from example["questions"]
+    else:
+        yield {k: example[k] for k in ("question", "question_type", "options", "gold_label") if k in example} | {
+            "name": example["task"]}
+
+
+def serialize_state(state: dict, utterance: str, n_turns: int) -> str:
+    """Regra única de serialização para todos os candidatos (state-text-v1)."""
+    lines = ["Contexto:",
+             f"Agente ativo: {state.get('active_agent') or 'nenhum'}"]
+    suspended = state.get("suspended_sessions") or []
+    lines.append("Sessões suspensas: " + (", ".join(suspended) if suspended else "nenhuma"))
+    turns = (state.get("recent_turns") or [])[-n_turns:] if n_turns > 0 else []
+    if turns:
+        lines.append("Turnos anteriores:")
+        for t in turns:
+            who = "Usuário" if t["role"] == "user" else f"Agente ({t.get('agent_id', '?')})"
+            lines.append(f"- {who}: {t['text']}")
+    lines += ["", f"Mensagem: {utterance}"]
+    return "\n".join(lines)
+
+
+def fit_state(state: dict, utterance: str, n_turns: int, budget: int,
+              count_tokens: Callable[[str], int]) -> tuple[str, dict]:
+    """Truncamento comum: descarta primeiro os turnos mais antigos; o enunciado só é
+    cortado no fim se sozinho estourar o orçamento."""
+    for n in range(n_turns, -1, -1):
+        text = serialize_state(state, utterance, n)
+        tokens = count_tokens(text)
+        if tokens <= budget:
+            return text, {"turns_used": n, "tokens": tokens, "truncated": n < n_turns}
+    lo, hi = 0, len(utterance)
+    while lo < hi:
+        mid = (lo + hi + 1) // 2
+        if count_tokens(serialize_state(state, utterance[:mid], 0)) <= budget:
+            lo = mid
+        else:
+            hi = mid - 1
+    text = serialize_state(state, utterance[:lo], 0)
+    return text, {"turns_used": 0, "tokens": count_tokens(text), "truncated": True,
+                  "utterance_chars_kept": lo}
+
+
+def jev_question(q: dict) -> dict:
+    kind = q.get("question_type", "choice")
+    if kind == "score":
+        criteria = list(q["options"].values()) if isinstance(q["options"], dict) else list(q["options"])
+    else:
+        criteria = dict(q["options"])
+    return {"type": kind, "instructions": q["question"], "criteria": criteria}
+
+
+def to_jev_request(example: dict, model: str, n_turns: int, budget: int | None = None,
+                   count_tokens: Callable[[str], int] | None = None) -> tuple[dict, dict]:
+    if budget is not None and count_tokens is not None:
+        state, info = fit_state(example["state"], example["utterance"], n_turns, budget, count_tokens)
+    else:
+        state, info = serialize_state(example["state"], example["utterance"], n_turns), {"truncated": False}
+    request = {"model": model, "state": state,
+               "questions": {q["name"]: jev_question(q) for q in iter_questions(example)}}
+    validate_request(request)
+    info["serialization"] = SERIALIZATION_VERSION
+    return request, info

dmb/jev.py

@@ -0,0 +1,128 @@
+"""Contrato jev-compat-v1: validação, erros padronizados e montagem das respostas (tarefas 37–40)."""
+from __future__ import annotations
+
+import json
+from functools import lru_cache
+from pathlib import Path
+
+import jsonschema
+
+CONTRACT_VERSION = "jev-compat-v1"
+SCHEMA_DIR = Path(__file__).resolve().parents[1] / "contracts" / "jev" / "schemas"
+# Piso numérico comum a todos os candidatos: o Laya arredonda em 4 casas e
+# probabilidades exatamente 0 tornariam a log-loss infinita.
+PROB_FLOOR = 1e-6
+
+
+class JevError(Exception):
+    status = 500
+    error_type = "internal"
+
+    def __init__(self, message: str):
+        super().__init__(message)
+        self.message = message
+
+    def body(self) -> dict:
+        return {"detail": {"error_type": self.error_type, "message": self.message}}
+
+
+class InvalidRequest(JevError):
+    status = 422
+    error_type = "invalid_request"
+
+
+class ContextLimit(JevError):
+    status = 422
+    error_type = "context"
+
+
+class UnknownOption(JevError):
+    status = 502
+    error_type = "unknown_option"
+
+
+class ModelUnavailable(JevError):
+    status = 503
+    error_type = "unavailable"
+
+
+@lru_cache(maxsize=None)
+def _validator(name: str) -> jsonschema.Draft202012Validator:
+    schema = json.loads((SCHEMA_DIR / f"{name}.schema.json").read_text())
+    return jsonschema.Draft202012Validator(schema)
+
+
+def validate_request(request: dict) -> None:
+    errors = sorted(_validator("request").iter_errors(request), key=lambda e: list(e.path))
+    if errors:
+        e = errors[0]
+        where = "/".join(str(p) for p in e.path) or "<raiz>"
+        raise InvalidRequest(f"{where}: {e.message}")
+
+
+def response_errors(response: dict) -> list[str]:
+    return [f"{'/'.join(map(str, e.path)) or '<raiz>'}: {e.message}"
+            for e in _validator("response").iter_errors(response)]
+
+
+def option_keys(question: dict) -> list[str]:
+    """Chaves das opções na ordem declarada (tarefa 49)."""
+    kind = question["type"]
+    if kind == "choice":
+        return list(question["criteria"])
+    if kind == "noul":
+        return ["true", "false"]
+    return [str(i) for i in range(len(question["criteria"]))]
+
+
+def option_descriptions(question: dict) -> dict[str, str]:
+    kind = question["type"]
+    if kind == "score":
+        return {str(i): d for i, d in enumerate(question["criteria"])}
+    return {k: question["criteria"][k] for k in option_keys(question)}
+
+
+def normalize(probabilities: dict[str, float], keys: list[str]) -> dict[str, float]:
+    """Mesmas chaves, mesma ordem, piso numérico e soma 1."""
+    if set(probabilities) != set(keys):
+        extra = sorted(set(probabilities) - set(keys))
+        missing = sorted(set(keys) - set(probabilities))
+        raise UnknownOption(f"opções divergentes: extras={extra} ausentes={missing}")
+    clipped = {k: max(float(probabilities[k]), PROB_FLOOR) for k in keys}
+    total = sum(clipped.values())
+    return {k: v / total for k, v in clipped.items()}
+
+
+def choice_answer(probabilities: dict[str, float], question: dict, confidence: float | None = None) -> dict:
+    keys = option_keys(question)
+    p = normalize(probabilities, keys)
+    choice = max(keys, key=p.__getitem__)
+    return {"type": "choice", "choice": choice,
+            "confidence": float(confidence) if confidence is not None else p[choice],
+            "probabilities": p}
+
+
+def noul_answer(p_true: float) -> dict:
+    p = min(max(float(p_true), PROB_FLOOR), 1 - PROB_FLOOR)
+    return {"type": "noul", "noul": p}
+
+
+def score_answer(probabilities: dict[str, float], question: dict, confidence: float | None = None) -> dict:
+    keys = option_keys(question)
+    p = normalize(probabilities, keys)
+    best = max(keys, key=p.__getitem__)
+    return {"type": "score", "score": sum(int(k) * v for k, v in p.items()),
+            "confidence": float(confidence) if confidence is not None else p[best],
+            "probabilities": p, "legend": option_descriptions(question)}
+
+
+def answer_from_distribution(question: dict, probabilities: dict[str, float],
+                             confidence: float | None = None) -> dict:
+    """Monta a resposta JEV de qualquer tipo a partir de uma distribuição sobre option_keys()."""
+    kind = question["type"]
+    if kind == "choice":
+        return choice_answer(probabilities, question, confidence)
+    if kind == "noul":
+        p = normalize(probabilities, ["true", "false"])
+        return noul_answer(p["true"])
+    return score_answer(probabilities, question, confidence)

dmb/llm.py

@@ -0,0 +1,91 @@
+"""Cliente mínimo do OCI Generative AI (chat on-demand) para geração e anotação de dados.
+
+Autenticação: resource principal (jobs/notebooks), instance principal (VM, DMB_OCI_AUTH=instance_principal)
+ou a chave de API em .secrets/ (máquina local).
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import time
+from functools import lru_cache
+from pathlib import Path
+
+GENAI_REGION = os.environ.get("DMB_GENAI_REGION", "us-chicago-1")
+ROOT = Path(__file__).resolve().parents[1]
+
+
+@lru_cache(maxsize=None)
+def _client_and_compartment():
+    import oci
+
+    endpoint = f"https://inference.generativeai.{GENAI_REGION}.oci.oraclecloud.com"
+    retry = oci.retry.DEFAULT_RETRY_STRATEGY
+    if os.environ.get("OCI_RESOURCE_PRINCIPAL_VERSION"):
+        signer = oci.auth.signers.get_resource_principals_signer()
+        client = oci.generative_ai_inference.GenerativeAiInferenceClient(
+            {}, signer=signer, service_endpoint=endpoint, retry_strategy=retry, timeout=(10, 300))
+    elif os.environ.get("DMB_OCI_AUTH") == "instance_principal":
+        signer = oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
+        client = oci.generative_ai_inference.GenerativeAiInferenceClient(
+            {}, signer=signer, service_endpoint=endpoint, retry_strategy=retry, timeout=(10, 300))
+    else:
+        sys.path.insert(0, str(ROOT / "infra" / "oci"))
+        from session import load_config
+        client = oci.generative_ai_inference.GenerativeAiInferenceClient(
+            load_config(region=GENAI_REGION), service_endpoint=endpoint, retry_strategy=retry, timeout=(10, 300))
+    compartment = os.environ.get("DMB_COMPARTMENT_OCID") or os.environ.get("JOB_COMPARTMENT_OCID")
+    if not compartment:
+        raise RuntimeError("defina DMB_COMPARTMENT_OCID")
+    return client, compartment
+
+
+def chat(model: str, system: str, user: str, temperature: float = 0.7, max_tokens: int = 4000,
+         attempts: int = 4) -> str:
+    import oci
+    from oci.generative_ai_inference import models as m
+
+    client, compartment = _client_and_compartment()
+    if model.startswith("cohere."):
+        request = m.CohereChatRequest(preamble_override=system, message=user, temperature=temperature,
+                                      max_tokens=max_tokens)
+    else:
+        msgs = [m.SystemMessage(content=[m.TextContent(text=system)]),
+                m.UserMessage(content=[m.TextContent(text=user)])]
+        kwargs = {"messages": msgs, "temperature": temperature}
+        # modelos de raciocínio da OpenAI só aceitam max_completion_tokens e temperatura padrão
+        if model.startswith("openai.gpt-5") or model.startswith("openai.o"):
+            kwargs.pop("temperature")
+            kwargs["max_completion_tokens"] = max_tokens
+        else:
+            kwargs["max_tokens"] = max_tokens
+        request = m.GenericChatRequest(**kwargs)
+    details = m.ChatDetails(compartment_id=compartment, serving_mode=m.OnDemandServingMode(model_id=model),
+                            chat_request=request)
+    for attempt in range(attempts):
+        try:
+            resp = client.chat(details).data.chat_response
+            if model.startswith("cohere."):
+                return resp.text
+            return "".join(c.text for c in resp.choices[0].message.content if getattr(c, "text", None))
+        except oci.exceptions.ServiceError as error:
+            if error.status in (429, 500, 502, 503, 504) and attempt < attempts - 1:
+                time.sleep(5 * 2 ** attempt)
+                continue
+            raise
+
+
+def extract_json(text: str):
+    """Primeiro objeto ou lista JSON do texto (tolera cercas ```json)."""
+    text = re.sub(r"^```(?:json)?|```$", "", text.strip(), flags=re.M).strip()
+    for opener, closer in (("[", "]"), ("{", "}")):
+        start = text.find(opener)
+        end = text.rfind(closer)
+        if start != -1 and end > start:
+            try:
+                return json.loads(text[start:end + 1])
+            except json.JSONDecodeError:
+                continue
+    raise ValueError(f"resposta sem JSON válido: {text[:200]!r}")

dmb/offline.py

@@ -0,0 +1,71 @@
+"""Isolamento de rede dentro do processo (tarefa 19, camada de aplicação).
+
+A camada de rede (VCN sem rota de saída) é a garantia principal; esta camada faz o
+processo falhar de forma explícita se um candidato tentar abrir conexão externa.
+"""
+from __future__ import annotations
+
+import ipaddress
+import os
+import socket
+
+OFFLINE_ENV = {
+    "HF_HUB_OFFLINE": "1",
+    "TRANSFORMERS_OFFLINE": "1",
+    "HF_DATASETS_OFFLINE": "1",
+    "HF_HUB_DISABLE_TELEMETRY": "1",
+    "USE_TF": "0",
+}
+
+
+class NetworkAccessBlocked(RuntimeError):
+    pass
+
+
+def _is_local(address) -> bool:
+    if isinstance(address, tuple):
+        host = address[0]
+    else:
+        return True  # sockets AF_UNIX
+    if host in ("localhost", ""):
+        return True
+    try:
+        return ipaddress.ip_address(host).is_loopback
+    except ValueError:
+        return False
+
+
+def enforce_offline() -> None:
+    os.environ.update(OFFLINE_ENV)
+    if getattr(socket.socket.connect, "_dmb_guard", False):
+        return
+    original_connect = socket.socket.connect
+    original_connect_ex = socket.socket.connect_ex
+
+    def guarded_connect(self, address):
+        if not _is_local(address):
+            raise NetworkAccessBlocked(f"conexão externa bloqueada durante o benchmark: {address}")
+        return original_connect(self, address)
+
+    def guarded_connect_ex(self, address):
+        if not _is_local(address):
+            raise NetworkAccessBlocked(f"conexão externa bloqueada durante o benchmark: {address}")
+        return original_connect_ex(self, address)
+
+    guarded_connect._dmb_guard = True
+    socket.socket.connect = guarded_connect
+    socket.socket.connect_ex = guarded_connect_ex
+
+
+def self_test() -> dict:
+    """Prova, dentro do processo, que o bloqueio está ativo: uma conexão externa tem de falhar."""
+    probe = socket.socket()
+    try:
+        probe.connect(("pypi.org", 443))
+        return {"process_guard": "FALHOU: conexão externa aberta"}
+    except NetworkAccessBlocked:
+        return {"process_guard": "ok"}
+    except OSError as error:  # sem rede nenhuma: também prova que não há saída
+        return {"process_guard": f"ok (rede indisponível: {type(error).__name__})"}
+    finally:
+        probe.close()

dmb/server.py

@@ -0,0 +1,93 @@
+"""Servidor local POST /v1/systemone, idêntico para todos os candidatos (tarefas 46–48, 50)."""
+from __future__ import annotations
+
+import json
+import os
+import time
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+
+from .adapter import Adapter
+from .jev import (CONTRACT_VERSION, InvalidRequest, JevError, ModelUnavailable, UnknownOption,
+                  option_keys, response_errors, validate_request)
+
+
+def build_response(adapter: Adapter, request: dict, answers: dict, input_tokens: int,
+                   latency_ms: float) -> dict:
+    if list(answers) != list(request["questions"]):
+        raise UnknownOption(f"perguntas respondidas {list(answers)} != {list(request['questions'])}")
+    for name, question in request["questions"].items():
+        answer = answers[name]
+        if answer.get("type") != question["type"]:
+            raise UnknownOption(f"{name}: tipo {answer.get('type')} != {question['type']}")
+        if question["type"] in ("choice", "score"):
+            keys = option_keys(question)
+            if list(answer["probabilities"]) != keys:
+                raise UnknownOption(f"{name}: ordem ou chaves das opções alteradas")
+            if question["type"] == "choice" and answer["choice"] not in keys:
+                raise UnknownOption(f"{name}: escolha {answer['choice']!r} fora de criteria")
+    response = {
+        "model": adapter.model_id,
+        "answers": answers,
+        "usage": {"input_tokens": int(input_tokens), "output_tokens": 0},
+        "metadata": {
+            "latency_ms": latency_ms,
+            "model_id": adapter.model_id,
+            "run_id": os.environ.get("DMB_RUN_ID", "local"),
+            "adapter_version": adapter.adapter_version,
+            "contract_version": CONTRACT_VERSION,
+            "probabilities_source": adapter.probabilities_source,
+        },
+    }
+    errors = response_errors(response)
+    if errors:
+        raise UnknownOption("resposta fora do contrato: " + "; ".join(errors[:3]))
+    return response
+
+
+def systemone(adapter: Adapter, request: dict) -> dict:
+    """Caminho único usado pelo HTTP e pelo harness em processo."""
+    validate_request(request)
+    state = request["state"]
+    if not isinstance(state, str):
+        # Todo adaptador recebe texto; estados estruturados viram JSON canônico.
+        state = json.dumps(state, ensure_ascii=False, sort_keys=True)
+    started = time.perf_counter()
+    answers, input_tokens = adapter.decide(state, request["questions"])
+    latency_ms = (time.perf_counter() - started) * 1000
+    return build_response(adapter, request, answers, input_tokens, latency_ms)
+
+
+def create_app(adapter: Adapter) -> FastAPI:
+    state = {"ready": False}
+
+    @asynccontextmanager
+    async def lifespan(_app):
+        started = time.perf_counter()
+        adapter.load()
+        state.update(ready=True, load_seconds=time.perf_counter() - started)
+        yield
+
+    app = FastAPI(title=f"dmb {adapter.name}", version=adapter.adapter_version, lifespan=lifespan)
+
+    @app.get("/health")
+    def health():
+        return {"status": "ready" if state["ready"] else "loading", **adapter.describe(),
+                "load_seconds": state.get("load_seconds")}
+
+    @app.post("/v1/systemone")
+    async def endpoint(http_request: Request):
+        try:
+            try:
+                body = await http_request.json()
+            except ValueError as error:
+                raise InvalidRequest(f"JSON inválido: {error}") from error
+            if not state["ready"]:
+                raise ModelUnavailable("modelo ainda carregando")
+            return systemone(adapter, body)
+        except JevError as error:
+            return JSONResponse(status_code=error.status, content=error.body())
+
+    return app