first commit

evaluator/judges/llm_judge.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+import json
+import re
+from collections import defaultdict
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationRecord, TraceJudgeResult, SessionJudgeResult
+from evaluator.llm.client import LLMClient, create_llm_client
+from evaluator.prompts.loader import load_prompt
+
+
+def _json_from_text(text: str) -> dict:
+    try:
+        return json.loads(text)
+    except Exception:
+        m = re.search(r"\{.*\}", text, flags=re.S)
+        if not m:
+            raise
+        return json.loads(m.group(0))
+
+
+def _history(record: ConversationRecord, turn_limit: int | None = None, max_chars: int = 6000) -> str:
+    messages = record.messages or []
+    if messages:
+        if turn_limit and turn_limit > 0:
+            # A turn is represented by one conversational message in the normalized record.
+            messages = messages[-turn_limit:]
+        text = "\n".join(f"{m.role}: {m.content}" for m in messages)
+    else:
+        text = f"user: {record.input_text}\nagent: {record.output_text}"
+    return text[-max_chars:]
+
+
+class TIMStyleLLMJudge:
+    def __init__(self, llm: LLMClient | None = None):
+        self.llm = llm or create_llm_client()
+        self.trace_prompt = load_prompt(settings.trace_prompt_path, 'trace_metrics')
+        self.session_prompt = load_prompt(settings.session_prompt_path, 'session_metrics')
+
+    @staticmethod
+    def _validated_trace_data(raw: str) -> dict:
+        data = _json_from_text(raw)
+        normalized = dict(data)
+        normalized.setdefault("judgeScore", normalized.get("judge_score"))
+        normalized.setdefault("accuracyScore", normalized.get("accuracy_score"))
+        normalized.setdefault("alucinationScore", normalized.get("alucination_score"))
+        normalized.setdefault("rationale", normalized.get("reasoning", ""))
+        # Pydantic validates the 0..1 contract here. Invalid metric output is treated
+        # like an invalid judge payload and receives the single specification retry.
+        TraceJudgeResult(**normalized)
+        return normalized
+
+    async def _trace_json_with_retry(self, prompt: str) -> tuple[dict | None, str | None]:
+        """One normal attempt + one retry-hint for invalid JSON/metric contract."""
+        raw = await self.llm.complete(prompt)
+        try:
+            return self._validated_trace_data(raw), None
+        except Exception as first_exc:
+            retry_prompt = f"""{prompt}
+
+RETRY-HINT:
+A resposta anterior não pôde ser interpretada como JSON válido ou violou o contrato das métricas.
+Retorne SOMENTE um objeto JSON válido, sem markdown, comentários ou texto adicional,
+com judgeScore, accuracyScore e alucinationScore entre 0.0 e 1.0 e a chave rationale.
+"""
+            retry_raw = await self.llm.complete(retry_prompt)
+            try:
+                return self._validated_trace_data(retry_raw), None
+            except Exception as second_exc:
+                return None, f"parse_failed_after_retry: {second_exc or first_exc}"
+
+    async def judge_trace(self, record: ConversationRecord) -> TraceJudgeResult:
+        prompt = f"""{self.trace_prompt}
+
+HISTÓRICO:
+{_history(record, turn_limit=settings.context_turn_limit)}
+
+MENSAGEM DO USUÁRIO:
+{record.input_text}
+
+RESPOSTA DO AGENTE:
+{record.output_text}
+
+METADATA:
+{json.dumps(record.metadata, ensure_ascii=False, default=str)}
+"""
+        data, parse_error = await self._trace_json_with_retry(prompt)
+        if data is None:
+            # Specification: after the second parse failure metrics become None;
+            # legacy exporter serializes those values as "0.00".
+            return TraceJudgeResult(
+                judgeScore=None,
+                accuracyScore=None,
+                alucinationScore=None,
+                rationale=parse_error or "parse_failed_after_retry",
+            )
+
+        data.setdefault("judge_name", "trace_metrics")
+        data.setdefault("judge_type", "trace")
+        data.setdefault("judgeScore", data.get("judge_score"))
+        data.setdefault("accuracyScore", data.get("accuracy_score"))
+        data.setdefault("alucinationScore", data.get("alucination_score"))
+        data.setdefault("rationale", data.get("reasoning", ""))
+        return TraceJudgeResult(**data)
+
+    async def judge_sessions(self, records: list[ConversationRecord]) -> dict[str, SessionJudgeResult]:
+        grouped: dict[str, list[ConversationRecord]] = defaultdict(list)
+        for r in records:
+            grouped[r.session_id].append(r)
+        out = {}
+        for session_id, items in grouped.items():
+            transcript = "\n".join(_history(r, max_chars=3000) for r in items)[-9000:]
+            prompt = f"""{self.session_prompt}
+
+TRANSCRIÇÃO DA SESSÃO:
+{transcript}
+"""
+            raw = await self.llm.complete(prompt)
+            data = _json_from_text(raw)
+            data.setdefault("judge_name", "session_metrics")
+            data.setdefault("judge_type", "session")
+            data.setdefault("inferredCsiScore", data.get("inferred_csi_score", 0))
+            data.setdefault("resolution", data.get("resolution", 0))
+            data.setdefault("conversationPrecision", data.get("conversation_precision", 0))
+            data.setdefault("rationale", data.get("reasoning", ""))
+            out[session_id] = SessionJudgeResult(**data)
+        return out