diff --git a/analysis/report.py b/analysis/report.py new file mode 100644 index 0000000..841d34c --- /dev/null +++ b/analysis/report.py @@ -0,0 +1,243 @@ +"""Análise do teste final (tarefas 67–74) a partir de reports/test/ (coletado com --all-reps). + +Qualidade e calibração usam a repetição 1 (as predições são determinísticas por candidato e +dispositivo; a concordância entre repetições é reportada). Desempenho usa as 3 repetições. + +Uso: python -m analysis.report reports/test → reports/test/analysis.json (+ confusion-*.csv, errors-*.jsonl) +""" +import json +import math +import random +import sys +from collections import Counter, defaultdict +from pathlib import Path + +import yaml + +from analysis.calibrate import apply_temperature, group_of +from analysis.metrics import calibration, classification, selective + +ROOT = Path(__file__).resolve().parents[1] +FALLBACK = "sem_correspondencia" +BOOT = 2000 +SEED = 20261007 +SLICES = ["profile", "n_options", "kind", "gold_eligible", "length_bucket", "sentiment", "domain", "annotation"] + + +def load_runs(src: Path) -> dict: + runs = defaultdict(dict) # (cand, dev) -> rep -> {"summary", "rows"} + for f in sorted(src.glob("summary-*-r*.json")): + s = json.loads(f.read_text()) + rep = int(f.stem.rsplit("-r", 1)[1]) + preds = f.with_name(f.name.replace("summary-", "predictions-").replace(".json", ".jsonl")) + rows = [json.loads(l) for l in preds.open()] if preds.exists() else [] + runs[(s["candidate"]["name"], s["device"])][rep] = {"summary": s, "rows": rows} + return runs + + +def calibrated(rows, t): + out = [] + for r in rows: + if r.get("probabilities"): + p = apply_temperature(r["probabilities"], t) + out.append({**r, "probabilities": p, "pred": max(p, key=p.get)}) + else: + out.append(r) + return out + + +def cluster_bootstrap(rows, stat, n=BOOT, seed=SEED): + groups = defaultdict(list) + for r in rows: + groups[group_of(r["id"])].append(r) + keys = sorted(groups) + rng = random.Random(seed) + values = [] + for _ in range(n): + sample = [r for k in rng.choices(keys, k=len(keys)) for r in groups[k]] + values.append(stat(sample)) + values.sort() + return values[int(0.025 * n)], values[int(0.975 * n) - 1] + + +def accuracy(rows): + ok = [r for r in rows if r.get("pred") is not None] + return sum(r["pred"] == r["gold"] for r in ok) / len(ok) if ok else 0.0 + + +def mcnemar(a_rows, b_rows): + """Teste exato de McNemar (binomial bilateral) sobre os pares discordantes.""" + b_by_id = {r["id"]: r for r in b_rows} + n01 = n10 = 0 + for r in a_rows: + o = b_by_id.get(r["id"]) + if not o: + continue + a_ok, b_ok = r["pred"] == r["gold"], o["pred"] == o["gold"] + n10 += a_ok and not b_ok + n01 += b_ok and not a_ok + n, k = n01 + n10, min(n01, n10) + p = min(1.0, 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n) if n else 1.0 + return {"a_only": n10, "b_only": n01, "p_value": p} + + +def paired_diff_ci(a_rows, b_rows): + b_by_id = {r["id"]: r for r in b_rows} + pairs = [(r, b_by_id[r["id"]]) for r in a_rows if r["id"] in b_by_id] + def diff(sample): + return sum((a["pred"] == a["gold"]) - (b["pred"] == b["gold"]) for a, b in sample) / len(sample) + groups = defaultdict(list) + for a, b in pairs: + groups[group_of(a["id"])].append((a, b)) + keys = sorted(groups) + rng = random.Random(SEED) + vals = sorted(diff([p for k in rng.choices(keys, k=len(keys)) for p in groups[k]]) for _ in range(BOOT)) + return diff(pairs), vals[int(0.025 * BOOT)], vals[int(0.975 * BOOT) - 1] + + +def error_class(r, taxonomy) -> str: + """Classificação automática dos erros (tarefa 73).""" + m = r["meta"] + gold, pred = r["gold"], r["pred"] + if r.get("truncated"): + return "truncamento" + if gold != FALLBACK and pred == FALLBACK: + return "abstencao_indevida" + if gold == FALLBACK and m.get("kind") == "oos": + return "fora_de_escopo_aceito" + if gold == FALLBACK and not m.get("gold_eligible", True): + return "intencao_fora_das_opcoes" + if pred in taxonomy.get(gold, {}).get("confusable_with", []) or gold in taxonomy.get(pred, {}).get("confusable_with", []): + return "taxonomia_par_dificil" + if m.get("ambiguity") in ("media", "alta") or m.get("annotation") == "desempate" or m.get("secondary"): + return "ambiguidade" + if m.get("profile") in ("erros_digitacao", "regionalismo", "informal_abreviado"): + return "linguagem" + return "desempenho_do_modelo" + + +def mean_sd(values): + values = [v for v in values if v is not None] + if not values: + return None, None + mu = sum(values) / len(values) + sd = (sum((v - mu) ** 2 for v in values) / (len(values) - 1)) ** 0.5 if len(values) > 1 else 0.0 + return mu, sd + + +def main(): + src = Path(sys.argv[1]) + cfg = yaml.safe_load((ROOT / "config" / "benchmark.yaml").read_text()) + cal = json.loads((ROOT / "config" / "calibration.json").read_text())["candidates"] + prices = yaml.safe_load((ROOT / "config" / "pricing.yaml").read_text())["usd_per_hour"] + manifest = json.loads((ROOT / "env" / "artifacts-manifest.json").read_text()) + frozen = json.loads((ROOT / "config" / "frozen.json").read_text()) + from datagen import taxonomy as tx + taxonomy = tx.load()["intents"] + runs = load_runs(src) + out = {"split": "test", "criteria": cfg["criteria"], "candidates": {}} + primary = {} + + for (cand, dev), reps in sorted(runs.items()): + r1 = reps[min(reps)] + raw = [r for r in r1["rows"] if not r.get("error")] + params = cal.get(cand, {}).get(dev) or cal.get(cand, {}).get("cuda") + cal_rows = calibrated(r1["rows"], params["temperature"]) + cls = classification(r1["rows"]) + lo, hi = cluster_bootstrap(r1["rows"], accuracy) + sel = selective([r for r in cal_rows if r.get("probabilities")], params["threshold"]) + lat = {k: mean_sd([rep["summary"]["latency_ms"][k] for rep in reps.values()]) for k in ("p50", "p95", "p99", "mean")} + agreement = None + if len(reps) > 1: + base = {r["id"]: r["pred"] for r in r1["rows"]} + agreement = min(sum(base.get(r["id"]) == r["pred"] for r in rep["rows"]) / len(rep["rows"]) + for rep in reps.values()) + mean_s = (lat["mean"][0] or 0) / 1000 + shape = "gpu_a10" if dev == "cuda" else "cpu_e4_8ocpu_64gb" + price = prices.get(shape) + art_bytes = sum(a["bytes"] for a in manifest["artifacts"] if a["candidate"] == cand and a["kind"] == "hf_model") + conf = Counter((r["gold"], r["pred"]) for r in r1["rows"] if r.get("pred") and r["pred"] != r["gold"]) + errors = [r for r in r1["rows"] if r.get("pred") and r["pred"] != r["gold"]] + err_classes = Counter(error_class(r, taxonomy) for r in errors) + slices = {} + for key in SLICES: + buckets = defaultdict(list) + for r in r1["rows"]: + buckets[str(r["n_options"] if key == "n_options" else r["meta"].get(key))].append(r) + slices[key] = {k: {"n": len(v), "accuracy": round(accuracy(v), 4)} for k, v in sorted(buckets.items())} + entry = { + "device": dev, "precision": r1["summary"].get("precision"), "repetitions": len(reps), + "quality": {"n": cls["n"], "errors": cls.get("errors", 0), "accuracy": round(cls["accuracy"], 4), + "accuracy_ci95": [round(lo, 4), round(hi, 4)], "macro_f1": round(cls["macro_f1"], 4), + "macro_precision": round(cls["macro_precision"], 4), "macro_recall": round(cls["macro_recall"], 4), + "fallback_recall": round(cls["per_label"].get(FALLBACK, {}).get("recall", 0), 4), + "fallback_precision": round(cls["per_label"].get(FALLBACK, {}).get("precision", 0), 4)}, + "calibration": {"temperature": params["temperature"], + "raw": {k: round(v, 4) for k, v in calibration(raw).items()}, + "calibrated": {k: round(v, 4) for k, v in calibration([r for r in cal_rows if r.get("probabilities")]).items()}}, + "abstention": {"threshold": params["threshold"], "coverage": round(sel["coverage"], 4), + "selective_accuracy": round(sel["selective_accuracy"] or 0, 4), + "confident_wrong_rate": round(sel["confident_wrong_rate"], 4)}, + "latency_ms": {k: {"mean": round(v[0], 2), "sd": round(v[1], 2)} for k, v in lat.items() if v[0] is not None}, + "throughput_decisions_per_s": round(1 / mean_s, 2) if mean_s else None, + "cost_usd_per_1k_decisions": round(price * mean_s * 1000 / 3600, 4) if price and mean_s else None, + "load_seconds": mean_sd([rep["summary"]["load_seconds"] for rep in reps.values()])[0], + "rss_peak_mib": max(rep["summary"].get("rss_peak_mib") or 0 for rep in reps.values()), + "gpu_mem_peak_mib": max(rep["summary"].get("gpu_mem_peak_mib") or 0 for rep in reps.values()) or None, + "storage_mib": {"weights": round(art_bytes / 2 ** 20), + "environment": round(frozen["environments"].get(cand, {}).get("bytes", 0) / 2 ** 20)}, + "prediction_agreement_across_reps": agreement, + "top_confusions": [{"gold": g, "pred": p, "n": n} for (g, p), n in conf.most_common(10)], + "error_classes": dict(err_classes.most_common()), + "slices": slices, + } + out["candidates"].setdefault(cand, {})[dev] = entry + primary[(cand, dev)] = r1["rows"] + with (src / f"confusion-{cand}-{dev}.csv").open("w") as f: + f.write("gold,pred,n\n" + "".join(f"{g},{p},{n}\n" for (g, p), n in conf.most_common())) + high_impact = sorted((r for r in cal_rows if r.get("pred") and r["pred"] != r["gold"] + and max(r["probabilities"].values()) >= params["threshold"]), + key=lambda r: -max(r["probabilities"].values())) + with (src / f"errors-high-impact-{cand}-{dev}.jsonl").open("w") as f: + for r in high_impact: + f.write(json.dumps({"id": r["id"], "gold": r["gold"], "pred": r["pred"], + "confidence": round(max(r["probabilities"].values()), 4), + "class": error_class(r, taxonomy), "profile": r["meta"].get("profile")}, + ensure_ascii=False) + "\n") + + # Comparações pareadas em GPU contra o candidato de maior acurácia (tarefa 74) + gpu = {c: rows for (c, d), rows in primary.items() if d == "cuda"} + if gpu: + best = max(gpu, key=lambda c: accuracy(gpu[c])) + out["pairwise_vs_best_gpu"] = {"best": best, "comparisons": {}} + for c, rows in gpu.items(): + if c == best: + continue + d, lo, hi = paired_diff_ci(gpu[best], rows) + out["pairwise_vs_best_gpu"]["comparisons"][c] = { + "accuracy_diff": round(d, 4), "ci95": [round(lo, 4), round(hi, 4)], **mcnemar(gpu[best], rows)} + # Critérios pré-registrados (Q5) + crit = cfg["criteria"] + out["criteria_check"] = {} + for cand, devs in out["candidates"].items(): + e = devs.get("cuda") + if not e: + continue + out["criteria_check"][cand] = { + "H3_latency_p95_gpu": e["latency_ms"]["p95"]["mean"] <= crit["latency_gpu_batch1"]["p95_ms_per_decision"], + "H1_macro_f1_absolute": e["quality"]["macro_f1"] >= crit["accuracy"]["fallback_macro_f1_min"], + "H2_risk": e["abstention"]["confident_wrong_rate"] <= crit["risk"]["max_confident_wrong_route_rate"] + and e["abstention"]["coverage"] >= crit["risk"]["min_coverage"], + } + (src / "analysis.json").write_text(json.dumps(out, indent=2, ensure_ascii=False) + "\n") + for cand, devs in out["candidates"].items(): + for dev, e in devs.items(): + q, a = e["quality"], e["abstention"] + print(f"{cand:10} {dev:4} acc {q['accuracy']:.3f} {q['accuracy_ci95']} f1 {q['macro_f1']:.3f} " + f"cov {a['coverage']:.3f} cwr {a['confident_wrong_rate']:.3f} p95 {e['latency_ms']['p95']} " + f"agree {e['prediction_agreement_across_reps']}") + print(json.dumps(out.get("pairwise_vs_best_gpu"), indent=1), json.dumps(out["criteria_check"], indent=1)) + + +if __name__ == "__main__": + main() diff --git a/config/pricing.yaml b/config/pricing.yaml new file mode 100644 index 0000000..39ef8e2 --- /dev/null +++ b/config/pricing.yaml @@ -0,0 +1,12 @@ +# Preços públicos OCI (pay-as-you-go, USD), consultados em 2026-10-07 na API pública de preços +# (apexapps.oracle.com/pls/apex/cetools/api/v1/products). Entrada só da análise de custo (tarefa 70); +# não afeta predições e por isso fica fora do congelamento. +source: oci-public-price-list +retrieved: 2026-10-07 +parts: + B95909: {name: "Compute - GPU - A10", unit: GPU-hora, usd: 2.00} + B93113: {name: "Compute - Standard - E4 - OCPU", unit: OCPU-hora, usd: 0.025} + B93114: {name: "Compute - Standard - E4 - Memory", unit: GB-hora, usd: 0.0015} +usd_per_hour: + gpu_a10: 2.00 # VM.GPU.A10.1 (1 GPU) + cpu_e4_8ocpu_64gb: 0.296 # 8 × 0,025 + 64 × 0,0015 diff --git a/infra/ds/collect.py b/infra/ds/collect.py index 622f276..2ffa028 100644 --- a/infra/ds/collect.py +++ b/infra/ds/collect.py @@ -27,6 +27,7 @@ def main(): ap.add_argument("--split", default="dev") ap.add_argument("--examples", type=int) ap.add_argument("--dest", required=True) + ap.add_argument("--all-reps", action="store_true", help="guarda cada repetição (DMB_REP) em separado") args = ap.parse_args() dest = ROOT / args.dest dest.mkdir(parents=True, exist_ok=True) @@ -45,21 +46,23 @@ def main(): continue if s.get("operational_env"): # execuções exploratórias com parâmetros alterados ficam de fora continue - key = (s["candidate"]["name"], s["device"]) + key = (s["candidate"]["name"], s["device"]) + ((s.get("repetition", 1),) if args.all_reps else ()) run_id = n.split("/")[1] if key not in latest or run_id > latest[key][0]: pred = n.replace("summary-", "predictions-").replace(".json", ".jsonl") latest[key] = (run_id, s, c, ns, bucket, pred, region) rows = [] - for (cand, dev), (run_id, s, c, ns, bucket, pred, region) in sorted(latest.items()): + for key, (run_id, s, c, ns, bucket, pred, region) in sorted(latest.items()): + cand, dev = key[0], key[1] + suffix = f"-{dev}-r{key[2]}" if args.all_reps else f"-{dev}" s["oci"] = {"region": region, "run_id": run_id} - (dest / f"summary-{cand}-{dev}.json").write_text(json.dumps(s, indent=2, ensure_ascii=False) + "\n") + (dest / f"summary-{cand}{suffix}.json").write_text(json.dumps(s, indent=2, ensure_ascii=False) + "\n") try: - (dest / f"predictions-{cand}-{dev}.jsonl").write_bytes(c.get_object(ns, bucket, pred).data.content) + (dest / f"predictions-{cand}{suffix}.jsonl").write_bytes(c.get_object(ns, bucket, pred).data.content) except oci.exceptions.ServiceError: pass q, lat, cal = s["quality"], s["latency_ms"], s.get("calibration_raw", {}) - rows.append(f"| {cand} | {dev} | {s.get('precision') or '—'} | {q.get('n', 0)} | " + rows.append(f"| {cand} | {dev}{' r' + str(key[2]) if args.all_reps else ''} | {s.get('precision') or '—'} | {q.get('n', 0)} | " f"{fmt(q.get('accuracy', 0) * 100)}% | {fmt(q.get('macro_f1', 0) * 100)}% | " f"{fmt(cal.get('ece'), 3)} | {fmt(lat['p50'])} | {fmt(lat['p95'])} | {fmt(lat['p99'])} | " f"{fmt(s['load_seconds'])} | {fmt(s.get('rss_peak_mib'), 0)} | {fmt(s.get('gpu_mem_peak_mib'), 0)} | "