first commit

evaluator/output/legacy_exporter.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import gzip
+import json
+from collections import defaultdict
+from pathlib import Path
+from datetime import datetime
+from zoneinfo import ZoneInfo
+from typing import Any
+
+from evaluator.config.settings import settings
+from evaluator.persistence.repository import EvaluationRepository
+from evaluator.analytics.vloop import max_consecutive_intent_streak
+
+HEADER = [
+    "judgeScore", "accuracyScore", "alucinationScore", "promptLength", "loop",
+    "inferredCsiScore", "resolution", "conversationPrecision",
+    "uraCallId", "channelId", "sessionId", "messageId"
+]
+
+
+def _q(v) -> str:
+    return '"' + str("" if v is None else v).replace('"', '""') + '"'
+
+
+def _metric(v) -> str | float | int:
+    # TIM specification: trace metric None must be serialized as "0.00".
+    return "0.00" if v is None else v
+
+
+def normalize_raw(raw):
+    if hasattr(raw, "read"):
+        raw = raw.read()
+    if isinstance(raw, bytes):
+        raw = raw.decode("utf-8")
+    if isinstance(raw, str):
+        raw = raw.strip()
+        if not raw:
+            return {}
+        raw = json.loads(raw)
+    if isinstance(raw, str):
+        raw = json.loads(raw)
+    return raw if isinstance(raw, dict) else {}
+
+
+def extract_prompt_length(raw: dict) -> int:
+    """TIM metric: raw character count of trace.input captured by the collector."""
+    nested = raw.get("raw") if isinstance(raw.get("raw"), dict) else {}
+    if "prompt_length" in nested:
+        try:
+            return int(nested.get("prompt_length") or 0)
+        except Exception:
+            pass
+    return len(str(raw.get("input_text") or ""))
+
+
+def extract_ura_call_id(raw: dict, metadata: dict | None = None, message_id: str | None = None) -> str:
+    metadata = metadata or {}
+    raw_metadata = raw.get("metadata") if isinstance(raw.get("metadata"), dict) else {}
+    business_context = (
+        metadata.get("business_context")
+        or metadata.get("businessContext")
+        or raw.get("business_context")
+        or raw.get("businessContext")
+        or raw_metadata.get("business_context")
+        or raw_metadata.get("businessContext")
+        or {}
+    )
+    if not isinstance(business_context, dict):
+        business_context = {}
+
+    nested = raw.get("raw") if isinstance(raw.get("raw"), dict) else {}
+    trace = nested.get("trace") if isinstance(nested.get("trace"), dict) else {}
+    detail = nested.get("detail") if isinstance(nested.get("detail"), dict) else {}
+    trace_input = trace.get("input") if isinstance(trace.get("input"), dict) else {}
+    detail_input = detail.get("input") if isinstance(detail.get("input"), dict) else {}
+    trace_metadata = trace.get("metadata") if isinstance(trace.get("metadata"), dict) else {}
+    detail_metadata = detail.get("metadata") if isinstance(detail.get("metadata"), dict) else {}
+    trace_bc = trace_input.get("business_context") if isinstance(trace_input.get("business_context"), dict) else {}
+    detail_bc = detail_input.get("business_context") if isinstance(detail_input.get("business_context"), dict) else {}
+
+    return str(
+        business_context.get("interaction_key")
+        or business_context.get("ura_call_id")
+        or metadata.get("ura_call_id")
+        or metadata.get("uraCallId")
+        or metadata.get("interaction_key")
+        or trace_metadata.get("ura_call_id")
+        or detail_metadata.get("ura_call_id")
+        or trace_bc.get("interaction_key")
+        or detail_bc.get("interaction_key")
+        or message_id
+        or ""
+    )
+
+
+def _session_loop_map(rows) -> dict[str, int]:
+    grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in rows:
+        session_id = row[1]
+        raw_json = row[8]
+        try:
+            grouped[str(session_id)].append(normalize_raw(raw_json))
+        except Exception:
+            grouped[str(session_id)].append({})
+    return {sid: max_consecutive_intent_streak(items) for sid, items in grouped.items()}
+
+
+def export_legacy_txt_gz(repo: EvaluationRepository, run_id: str, agent_id: str) -> Path:
+    output_dir = settings.path(settings.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    now_brt = datetime.now(ZoneInfo("America/Sao_Paulo"))
+    path = output_dir / f"AGENTE_{agent_id}_LLM_JUDGE_{now_brt.strftime('%Y%m%d_%H%M%S')}.TXT.GZ"
+
+    with repo.store.connect() as conn:
+        cur = conn.cursor()
+        cur.execute(f"""
+            select SESSION_ID, INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION
+              from {repo.store.t('EVALUATION_RESULT')}
+             where RUN_ID = :run_id and JUDGE_TYPE = 'SESSION'
+        """, {"run_id": run_id})
+        session_metrics = {
+            sid: {"inferredCsiScore": csi, "resolution": res, "conversationPrecision": prec}
+            for sid, csi, res, prec in cur.fetchall()
+        }
+
+        cur.execute(f"""
+            select r.TRACE_ID, r.SESSION_ID, r.JUDGE_SCORE, r.ACCURACY_SCORE,
+                   r.ALUCINATION_SCORE, r.RATIONALE, i.CHANNEL, i.MESSAGE_ID, i.RAW_JSON
+              from {repo.store.t('EVALUATION_RESULT')} r
+              left join {repo.store.t('EVALUATION_ITEM')} i on i.ITEM_ID = r.ITEM_ID
+             where r.RUN_ID = :run_id and r.JUDGE_TYPE = 'TRACE'
+             order by r.CREATED_AT
+        """, {"run_id": run_id})
+        rows = cur.fetchall()
+
+    loop_by_session = _session_loop_map(rows)
+
+    with gzip.open(path, "wt", encoding="utf-8") as f:
+        f.write("|;".join(_q(v) for v in HEADER) + "\n")
+        for trace_id, session_id, judge, accuracy, alucination, rationale, channel, message_id, raw_json in rows:
+            session = session_metrics.get(session_id, {})
+            raw = {}
+            ura_call_id = ""
+            channel_id = channel or ""
+            prompt_length = 0
+            try:
+                raw = normalize_raw(raw_json)
+                metadata = raw.get("metadata") if isinstance(raw.get("metadata"), dict) else {}
+                channel_id = (
+                    metadata.get("channel_id")
+                    or metadata.get("channelId")
+                    or metadata.get("channel")
+                    or channel_id
+                )
+                ura_call_id = extract_ura_call_id(raw, metadata, message_id)
+                prompt_length = extract_prompt_length(raw)
+            except Exception as exc:
+                print(f"[legacy_exporter] metadata extraction failed trace_id={trace_id}: {exc}")
+
+            vals = [
+                _metric(judge),
+                _metric(accuracy),
+                _metric(alucination),
+                prompt_length,
+                loop_by_session.get(str(session_id), 0),
+                session.get("inferredCsiScore"),
+                session.get("resolution"),
+                session.get("conversationPrecision"),
+                ura_call_id,
+                channel_id,
+                session_id,
+                message_id or trace_id,
+            ]
+            f.write("|;".join(_q(v) for v in vals) + "\n")
+        f.write("|;".join([_q("TOTAL"), _q(len(rows))]) + "\n")
+    return path