update config and docs

analysis/metrics.py

@@ -0,0 +1,77 @@
+"""Métricas de qualidade e desempenho (tarefas 67–69)."""
+from __future__ import annotations
+
+import math
+from collections import Counter, defaultdict
+
+FALLBACK = "sem_correspondencia"
+
+
+def percentile(values: list[float], q: float) -> float | None:
+    if not values:
+        return None
+    v = sorted(values)
+    k = (len(v) - 1) * q
+    lo, hi = math.floor(k), math.ceil(k)
+    return v[lo] + (v[hi] - v[lo]) * (k - lo)
+
+
+def latency_summary(values: list[float]) -> dict:
+    return {"n": len(values), "p50": percentile(values, 0.5), "p95": percentile(values, 0.95),
+            "p99": percentile(values, 0.99), "mean": sum(values) / len(values) if values else None}
+
+
+def classification(rows: list[dict]) -> dict:
+    """rows: dicts com `gold`, `pred` (rótulos). Ignora linhas com erro (pred None)."""
+    ok = [r for r in rows if r.get("pred") is not None]
+    if not ok:
+        return {"n": 0}
+    labels = sorted({r["gold"] for r in ok} | {r["pred"] for r in ok})
+    tp, fp, fn = Counter(), Counter(), Counter()
+    for r in ok:
+        if r["pred"] == r["gold"]:
+            tp[r["gold"]] += 1
+        else:
+            fp[r["pred"]] += 1
+            fn[r["gold"]] += 1
+    per = {}
+    for lab in labels:
+        p = tp[lab] / (tp[lab] + fp[lab]) if tp[lab] + fp[lab] else 0.0
+        rc = tp[lab] / (tp[lab] + fn[lab]) if tp[lab] + fn[lab] else 0.0
+        per[lab] = {"precision": p, "recall": rc, "f1": 2 * p * rc / (p + rc) if p + rc else 0.0,
+                    "support": tp[lab] + fn[lab]}
+    gold_labels = [lab for lab in labels if per[lab]["support"] > 0]
+    return {
+        "n": len(ok), "errors": len(rows) - len(ok),
+        "accuracy": sum(r["pred"] == r["gold"] for r in ok) / len(ok),
+        "macro_f1": sum(per[lab]["f1"] for lab in gold_labels) / len(gold_labels),
+        "macro_precision": sum(per[lab]["precision"] for lab in gold_labels) / len(gold_labels),
+        "macro_recall": sum(per[lab]["recall"] for lab in gold_labels) / len(gold_labels),
+        "per_label": per,
+    }
+
+
+def calibration(rows: list[dict], bins: int = 15) -> dict:
+    """rows: `probabilities` (dict), `gold`, `pred`. Brier multiclasse e ECE sobre a confiança max(p)."""
+    ok = [r for r in rows if r.get("probabilities")]
+    if not ok:
+        return {}
+    brier = sum(sum((p - (k == r["gold"])) ** 2 for k, p in r["probabilities"].items()) for r in ok) / len(ok)
+    nll = -sum(math.log(max(r["probabilities"].get(r["gold"], 0.0), 1e-12)) for r in ok) / len(ok)
+    buckets = defaultdict(list)
+    for r in ok:
+        conf = max(r["probabilities"].values())
+        buckets[min(int(conf * bins), bins - 1)].append((conf, r["pred"] == r["gold"]))
+    ece = sum(len(b) / len(ok) * abs(sum(c for c, _ in b) / len(b) - sum(a for _, a in b) / len(b))
+              for b in buckets.values())
+    return {"brier": brier, "nll": nll, "ece": ece}
+
+
+def selective(rows: list[dict], threshold: float) -> dict:
+    """Abstém (encaminha para desambiguação) quando max(p) < limiar."""
+    ok = [r for r in rows if r.get("probabilities")]
+    kept = [r for r in ok if max(r["probabilities"].values()) >= threshold]
+    wrong_confident = sum(r["pred"] != r["gold"] for r in kept)
+    return {"threshold": threshold, "coverage": len(kept) / len(ok) if ok else None,
+            "selective_accuracy": (len(kept) - wrong_confident) / len(kept) if kept else None,
+            "confident_wrong_rate": wrong_confident / len(ok) if ok else None}