mirror of
https://github.com/mcloh/decision-models-benchmark.git
synced 2026-11-05 22:04:12 +00:00
78 lines
3.4 KiB
Python
78 lines
3.4 KiB
Python
"""Métricas de qualidade e desempenho (tarefas 67–69)."""
|
||
from __future__ import annotations
|
||
|
||
import math
|
||
from collections import Counter, defaultdict
|
||
|
||
FALLBACK = "sem_correspondencia"
|
||
|
||
|
||
def percentile(values: list[float], q: float) -> float | None:
|
||
if not values:
|
||
return None
|
||
v = sorted(values)
|
||
k = (len(v) - 1) * q
|
||
lo, hi = math.floor(k), math.ceil(k)
|
||
return v[lo] + (v[hi] - v[lo]) * (k - lo)
|
||
|
||
|
||
def latency_summary(values: list[float]) -> dict:
|
||
return {"n": len(values), "p50": percentile(values, 0.5), "p95": percentile(values, 0.95),
|
||
"p99": percentile(values, 0.99), "mean": sum(values) / len(values) if values else None}
|
||
|
||
|
||
def classification(rows: list[dict]) -> dict:
|
||
"""rows: dicts com `gold`, `pred` (rótulos). Ignora linhas com erro (pred None)."""
|
||
ok = [r for r in rows if r.get("pred") is not None]
|
||
if not ok:
|
||
return {"n": 0}
|
||
labels = sorted({r["gold"] for r in ok} | {r["pred"] for r in ok})
|
||
tp, fp, fn = Counter(), Counter(), Counter()
|
||
for r in ok:
|
||
if r["pred"] == r["gold"]:
|
||
tp[r["gold"]] += 1
|
||
else:
|
||
fp[r["pred"]] += 1
|
||
fn[r["gold"]] += 1
|
||
per = {}
|
||
for lab in labels:
|
||
p = tp[lab] / (tp[lab] + fp[lab]) if tp[lab] + fp[lab] else 0.0
|
||
rc = tp[lab] / (tp[lab] + fn[lab]) if tp[lab] + fn[lab] else 0.0
|
||
per[lab] = {"precision": p, "recall": rc, "f1": 2 * p * rc / (p + rc) if p + rc else 0.0,
|
||
"support": tp[lab] + fn[lab]}
|
||
gold_labels = [lab for lab in labels if per[lab]["support"] > 0]
|
||
return {
|
||
"n": len(ok), "errors": len(rows) - len(ok),
|
||
"accuracy": sum(r["pred"] == r["gold"] for r in ok) / len(ok),
|
||
"macro_f1": sum(per[lab]["f1"] for lab in gold_labels) / len(gold_labels),
|
||
"macro_precision": sum(per[lab]["precision"] for lab in gold_labels) / len(gold_labels),
|
||
"macro_recall": sum(per[lab]["recall"] for lab in gold_labels) / len(gold_labels),
|
||
"per_label": per,
|
||
}
|
||
|
||
|
||
def calibration(rows: list[dict], bins: int = 15) -> dict:
|
||
"""rows: `probabilities` (dict), `gold`, `pred`. Brier multiclasse e ECE sobre a confiança max(p)."""
|
||
ok = [r for r in rows if r.get("probabilities")]
|
||
if not ok:
|
||
return {}
|
||
brier = sum(sum((p - (k == r["gold"])) ** 2 for k, p in r["probabilities"].items()) for r in ok) / len(ok)
|
||
nll = -sum(math.log(max(r["probabilities"].get(r["gold"], 0.0), 1e-12)) for r in ok) / len(ok)
|
||
buckets = defaultdict(list)
|
||
for r in ok:
|
||
conf = max(r["probabilities"].values())
|
||
buckets[min(int(conf * bins), bins - 1)].append((conf, r["pred"] == r["gold"]))
|
||
ece = sum(len(b) / len(ok) * abs(sum(c for c, _ in b) / len(b) - sum(a for _, a in b) / len(b))
|
||
for b in buckets.values())
|
||
return {"brier": brier, "nll": nll, "ece": ece}
|
||
|
||
|
||
def selective(rows: list[dict], threshold: float) -> dict:
|
||
"""Abstém (encaminha para desambiguação) quando max(p) < limiar."""
|
||
ok = [r for r in rows if r.get("probabilities")]
|
||
kept = [r for r in ok if max(r["probabilities"].values()) >= threshold]
|
||
wrong_confident = sum(r["pred"] != r["gold"] for r in kept)
|
||
return {"threshold": threshold, "coverage": len(kept) / len(ok) if ok else None,
|
||
"selective_accuracy": (len(kept) - wrong_confident) / len(kept) if kept else None,
|
||
"confident_wrong_rate": wrong_confident / len(ok) if ok else None}
|