first commit

evaluator/analytics/vloop.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any, Iterable
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    return value if isinstance(value, dict) else {}
+
+
+def extract_intent(record_or_raw: Any) -> str | None:
+    """Extract the trace intent used by the TIM specification for the deterministic loop metric."""
+    if isinstance(record_or_raw, dict):
+        metadata = _as_dict(record_or_raw.get("metadata"))
+        nested_raw = _as_dict(record_or_raw.get("raw"))
+    else:
+        metadata = _as_dict(getattr(record_or_raw, "metadata", None))
+        nested_raw = _as_dict(getattr(record_or_raw, "raw", None))
+
+    trace = _as_dict(nested_raw.get("trace"))
+    detail = _as_dict(nested_raw.get("detail"))
+    trace_meta = _as_dict(trace.get("metadata"))
+    detail_meta = _as_dict(detail.get("metadata"))
+
+    value = (
+        metadata.get("intent")
+        or trace_meta.get("intent")
+        or detail_meta.get("intent")
+    )
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+def max_consecutive_intent_streak(records: Iterable[Any]) -> int:
+    """Maximum consecutive streak of the same metadata.intent in trace order."""
+    maximum = 0
+    current = 0
+    previous: str | None = None
+
+    for record in records:
+        intent = extract_intent(record)
+        if not intent:
+            previous = None
+            current = 0
+            continue
+        if intent == previous:
+            current += 1
+        else:
+            previous = intent
+            current = 1
+        maximum = max(maximum, current)
+
+    return maximum
+
+
+def session_loop_values(records: Iterable[Any], session_id_getter=None) -> dict[str, int]:
+    grouped: dict[str, list[Any]] = defaultdict(list)
+    for record in records:
+        if session_id_getter:
+            sid = session_id_getter(record)
+        elif isinstance(record, dict):
+            sid = record.get("session_id") or record.get("sessionId")
+        else:
+            sid = getattr(record, "session_id", None)
+        if sid:
+            grouped[str(sid)].append(record)
+    return {sid: max_consecutive_intent_streak(items) for sid, items in grouped.items()}
+
+
+# Backward-compatible name used by older integrations. For a single trace the
+# streak is either 0 (no intent) or 1 (intent present); the official metric is
+# session-level and should use max_consecutive_intent_streak over all traces.
+def vloop_flag(record_or_raw: Any) -> int:
+    return max_consecutive_intent_streak([record_or_raw])

evaluator/cli.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+import asyncio
+from datetime import datetime, timedelta
+import typer
+from zoneinfo import ZoneInfo
+from rich import print
+from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeElapsedColumn
+from evaluator.config.agents import load_agents
+from evaluator.engine import EvaluationEngine
+from evaluator.persistence.repository import EvaluationRepository
+from evaluator.config.settings import settings
+from evaluator.time_utils import brt_day_window
+
+app = typer.Typer(help='Agent Framework TIM-style LLM Judge Evaluator')
+
+
+def _run_progress(coro_factory):
+    async def runner():
+        state={'run_id': None}
+        with Progress(SpinnerColumn(), TextColumn('[bold blue]{task.fields[stage]}'), BarColumn(), TextColumn('{task.completed}/{task.total}'), TextColumn('{task.percentage:>3.0f}%'), TimeElapsedColumn()) as progress:
+            task=progress.add_task('evaluation', total=1, stage='starting')
+            async def cb(event):
+                state['run_id'] = event.get('run_id') or state['run_id']
+                stage = event.get('stage','')
+                msg = event.get('message','')
+                if state['run_id']:
+                    snap = await EvaluationRepository(auto_init_schema=False).aget_run_progress(state['run_id'], event_limit=1)
+                    total=int(snap.get('total_items') or 0) or 1
+                    done=int(snap.get('done_items') or 0)
+                    progress.update(task,total=total,completed=done,stage=f'{stage}: {msg}'[:120])
+            result = await coro_factory(cb)
+            progress.update(task, completed=1, total=1, stage='finished')
+            return result
+    return asyncio.run(runner())
+
+@app.command("reset-db")
+def reset_db():
+    repo = EvaluationRepository(auto_init_schema=False)
+    repo.store.drop_schema()
+    repo.store._init_schema()
+    print({"status": "OK", "message": "Evaluator schema dropped and recreated successfully."})
+
+@app.command('init-db')
+def init_db():
+    EvaluationRepository(auto_init_schema=True)
+    print({'status':'OK','message':'schema checked/created'})
+
+@app.command('show-config')
+def show_config():
+    print({'env_path': str(settings.project_root / '.env'), 'adb_dsn': settings.ADB_DSN, 'wallet': settings.ADB_WALLET_LOCATION, 'langfuse': settings.enable_langfuse, 'publish_langfuse_scores': settings.publish_langfuse_scores, 'llm_provider': settings.llm_provider, 'llm_profile': settings.llm_profile, 'oci_genai_base_url': settings.OCI_GENAI_BASE_URL, 'oci_genai_model': settings.OCI_GENAI_MODEL, 'oci_genai_api_key_configured': bool(settings.OCI_GENAI_API_KEY), 'agents_config': settings.agents_config_path})
+
+@app.command('run')
+def run(period_start: datetime, period_end: datetime, source: str='langfuse', limit: int|None=None, show_progress: bool=True):
+    if show_progress:
+        result = _run_progress(lambda cb: EvaluationEngine(progress_callback=cb).run(period_start, period_end, source, limit))
+    else:
+        result = asyncio.run(EvaluationEngine().run(period_start, period_end, source, limit))
+    print(result)
+
+@app.command('run-agents')
+def run_agents(source: str='langfuse', agent_id: str|None=None, limit: int|None=None):
+    async def main():
+        results=[]
+        today_brt = datetime.now(ZoneInfo("America/Sao_Paulo")).date()
+        for agent in load_agents():
+            if agent_id and agent.agent_id != agent_id:
+                continue
+            reference_date = today_brt - timedelta(days=agent.days_back)
+            start, end = brt_day_window(reference_date)
+            announced = {"run_id": None}
+            async def announce_run(event, _agent_id=agent.agent_id):
+                run_id = event.get("run_id")
+                if run_id and announced["run_id"] != run_id:
+                    announced["run_id"] = run_id
+                    print(f"[bold cyan][{_agent_id}][/bold cyan] RUN_ID=[bold]{run_id}[/bold]")
+                    print(f"  progress: python -m evaluator.cli progress {run_id} --events 100")
+            engine=EvaluationEngine(progress_callback=announce_run)
+            results.append(await engine.run_agent(
+                agent, start, end, source=source, limit=limit, reference_date=reference_date
+            ))
+        return results
+    print(asyncio.run(main()))
+
+@app.command('progress')
+def progress(run_id: str, events: int=20):
+    print(asyncio.run(EvaluationRepository(auto_init_schema=False).aget_run_progress(run_id, event_limit=events)))
+
+@app.command('runs')
+def runs(limit: int=20):
+    print(asyncio.run(EvaluationRepository(auto_init_schema=False).alist_runs(limit)))
+
+if __name__ == '__main__':
+    app()

evaluator/collectors/agent_framework.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+from datetime import datetime
+from evaluator.collectors.base import ConversationCollector
+from evaluator.core.models import ConversationRecord, ConversationMessage
+from evaluator.persistence.oracle_store import OracleStore, _json_loads
+from evaluator.config.settings import settings
+
+class AgentFrameworkCollector(ConversationCollector):
+    def __init__(self):
+        self.store = OracleStore(settings, auto_init_schema=False)
+
+    async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None):
+        return await self.store.to_thread(self._collect, period_start, period_end, agent_aliases or set(), limit or 100)
+
+    def _collect(self, period_start, period_end, aliases, limit):
+        records=[]
+        with self.store.connect() as conn:
+            cur=conn.cursor()
+            cur.execute(f"""
+                select * from (
+                    select SESSION_ID, AGENT_ID, CHANNEL, CONTEXT_JSON, METADATA_JSON, CREATED_AT
+                      from {self.store.t('AGENT_SESSION')}
+                     where CREATED_AT >= :start_at and CREATED_AT < :end_at
+                     order by CREATED_AT desc
+                ) where rownum <= :max_rows
+            """, dict(start_at=period_start, end_at=period_end, max_rows=limit))
+            sessions=cur.fetchall()
+            for session_id, agent_id, channel, ctx, meta, created_at in sessions:
+                if aliases and agent_id not in aliases: continue
+                cur.execute(f"""
+                    select ROLE, CONTENT, METADATA_JSON, CREATED_AT, MESSAGE_ID
+                      from {self.store.t('AGENT_MESSAGE')}
+                     where SESSION_ID=:session_id order by CREATED_AT
+                """, dict(session_id=session_id))
+                rows=cur.fetchall()
+                msgs=[]
+                for role, content, msg_meta, msg_created, message_id in rows:
+                    msgs.append(ConversationMessage(role=role, content=content or '', created_at=str(msg_created), metadata=_json_loads(msg_meta.read() if hasattr(msg_meta,'read') else msg_meta,{})))
+                input_text=next((m.content for m in msgs if m.role in ('user','human')), '')
+                output_text=next((m.content for m in reversed(msgs) if m.role in ('assistant','ai','agent')), '')
+                records.append(ConversationRecord(session_id=session_id, trace_id=session_id, message_id=rows[-1][4] if rows else None, agent_id=agent_id, channel=channel, input_text=input_text, output_text=output_text, messages=msgs, metadata=_json_loads(meta.read() if hasattr(meta,'read') else meta,{}), raw={}))
+        return records

evaluator/collectors/base.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from datetime import datetime
+from evaluator.core.models import ConversationRecord
+
+class ConversationCollector(ABC):
+    @abstractmethod
+    async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None) -> list[ConversationRecord]: ...

evaluator/collectors/langfuse.py

@@ -0,0 +1,518 @@
+from __future__ import annotations
+
+import json
+import math
+import random
+from datetime import datetime
+from typing import Any, Iterable
+
+import httpx
+
+from evaluator.collectors.base import ConversationCollector
+from evaluator.collectors.langfuse_credentials import LangfuseCredentials, resolve_langfuse_credentials
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationMessage, ConversationRecord
+from evaluator.identity.resolver import IdentityResolver
+from evaluator.time_utils import to_utc_z
+
+
+def _iso_z(dt: datetime) -> str:
+    return to_utc_z(dt)
+
+
+def _metadata(obj: dict[str, Any] | None) -> dict[str, Any]:
+    if not isinstance(obj, dict):
+        return {}
+    meta = obj.get("metadata") or {}
+    return meta if isinstance(meta, dict) else {}
+
+
+def _first_value(*values: Any) -> Any:
+    for value in values:
+        if value not in (None, "", [], {}):
+            return value
+    return None
+
+
+def _content_to_text(value: Any) -> str:
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (int, float, bool)):
+        return str(value)
+    if isinstance(value, list):
+        return "\n".join(filter(None, (_content_to_text(v) for v in value)))
+    if isinstance(value, dict):
+        for key in ("text", "content", "message", "value", "input", "output", "completion"):
+            if key in value:
+                text = _content_to_text(value.get(key))
+                if text:
+                    return text
+        choices = value.get("choices")
+        if isinstance(choices, list) and choices:
+            return _content_to_text(choices[0])
+        return ""
+    return str(value)
+
+
+def _raw_character_count(value: Any) -> int:
+    """Character-count equivalent of len(trace.input) without losing structured input."""
+    if value is None:
+        return 0
+    if isinstance(value, str):
+        return len(value)
+    try:
+        return len(json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str))
+    except Exception:
+        return len(str(value))
+
+
+def _messages_from_value(value: Any, default_role: str) -> list[ConversationMessage]:
+    if value in (None, "", [], {}):
+        return []
+    if isinstance(value, str):
+        text = value.strip()
+        return [ConversationMessage(role=default_role, content=text)] if text else []
+    if isinstance(value, dict):
+        for key in ("messages", "conversation", "chat"):
+            if isinstance(value.get(key), list):
+                return _messages_from_value(value[key], default_role)
+        if "role" in value and "content" in value:
+            role = str(value.get("role") or default_role)
+            content = _content_to_text(value.get("content")).strip()
+            return [ConversationMessage(role=role, content=content, metadata={"source": "langfuse"})] if content else []
+        text = _content_to_text(value).strip()
+        return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else []
+    if isinstance(value, list):
+        out: list[ConversationMessage] = []
+        for item in value:
+            out.extend(_messages_from_value(item, default_role))
+        return out
+    text = _content_to_text(value).strip()
+    return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else []
+
+
+def _agent_id(trace: dict[str, Any]) -> str | None:
+    meta = _metadata(trace)
+    return meta.get("agent_id") or meta.get("agentId") or meta.get("agent") or trace.get("name")
+
+
+def _trace_sort_key(trace: dict[str, Any]) -> str:
+    return str(trace.get("timestamp") or trace.get("createdAt") or trace.get("created_at") or trace.get("startTime") or "")
+
+
+def _observation_sort_key(obs: dict[str, Any]) -> str:
+    return str(obs.get("startTime") or obs.get("start_time") or obs.get("createdAt") or obs.get("timestamp") or "")
+
+
+def _session_entity_id(obj: dict[str, Any]) -> str | None:
+    """Return the identifier of a Langfuse Session entity.
+
+    The generic ``id`` fallback is valid only for objects returned by
+    ``/api/public/sessions``. It must never be used for traces because a trace
+    id is not a conversation/session id.
+    """
+    value = obj.get("sessionId") or obj.get("session_id") or obj.get("id")
+    return str(value) if value else None
+
+
+def _session_id_from_metadata(obj: dict[str, Any] | None) -> str | None:
+    meta = _metadata(obj)
+    value = _first_value(
+        meta.get("session_id"),
+        meta.get("sessionId"),
+        meta.get("conversation_key"),
+        meta.get("conversationKey"),
+    )
+    if value:
+        return str(value)
+
+    business_context = meta.get("business_context")
+    if isinstance(business_context, dict):
+        value = _first_value(
+            business_context.get("session_id"),
+            business_context.get("sessionId"),
+            business_context.get("conversation_key"),
+            business_context.get("conversationKey"),
+            business_context.get("session_key"),
+            business_context.get("sessionKey"),
+        )
+        if value:
+            return str(value)
+    return None
+
+
+def _trace_session_id(trace: dict[str, Any], observations: list[dict[str, Any]] | None = None) -> str | None:
+    """Resolve the real conversation session id for a trace.
+
+    Precedence:
+      1. Native Langfuse trace.sessionId/session_id.
+      2. Trace metadata session/conversation identifiers.
+      3. Observation metadata session/conversation identifiers.
+
+    Deliberately *never* falls back to trace.id.
+    """
+    value = trace.get("sessionId") or trace.get("session_id")
+    if value:
+        return str(value)
+
+    value = _session_id_from_metadata(trace)
+    if value:
+        return value
+
+    for obs in observations or []:
+        value = _session_id_from_metadata(obs)
+        if value:
+            return value
+    return None
+
+
+def _recovered_session(trace: dict[str, Any], observations: list[dict[str, Any]], session_id: str) -> dict[str, Any]:
+    """Build minimal session context for historical traces lacking native Langfuse sessions."""
+    merged_meta: dict[str, Any] = {}
+    for obs in observations:
+        merged_meta.update(_metadata(obs))
+    # Trace metadata has precedence over observation-derived fallback metadata.
+    merged_meta.update(_metadata(trace))
+    merged_meta.setdefault("session_id", session_id)
+    return {"id": session_id, "sessionId": session_id, "metadata": merged_meta, "recovered": True}
+
+
+def _items(payload: Any, keys: Iterable[str]) -> list[dict[str, Any]]:
+    if isinstance(payload, list):
+        return [x for x in payload if isinstance(x, dict)]
+    if not isinstance(payload, dict):
+        return []
+    for key in keys:
+        value = payload.get(key)
+        if isinstance(value, list):
+            return [x for x in value if isinstance(x, dict)]
+    data = payload.get("data")
+    if isinstance(data, list):
+        return [x for x in data if isinstance(x, dict)]
+    return []
+
+
+class LangfuseCollector(ConversationCollector):
+    """TIM-compliant Langfuse v3 collector.
+
+    Collection is session-oriented and bulk/paginated:
+      1. Read every native Langfuse session in the BRT-derived UTC window.
+      2. Read every trace and observation in the same window, once, page by page.
+      3. Recover historical session ids from trace/observation metadata when the
+         native Langfuse session relation is absent.
+      4. Deterministically sample the resulting logical session ids.
+      5. Keep all traces belonging to sampled sessions.
+
+    This preserves session-level semantics while eliminating the previous
+    trace-detail + observations N+1 pattern.
+    """
+
+    PAGE_SIZE = 100
+
+    def __init__(self, agent_id: str | None = None):
+        self.identity_resolver = IdentityResolver(settings.identity_config_path)
+        self.agent_id = agent_id
+
+    async def _get_page(self, client: httpx.AsyncClient, path: str, params: dict[str, Any], creds: LangfuseCredentials) -> Any:
+        response = await client.get(path, params=params, auth=creds.auth)
+        if response.status_code >= 400:
+            raise RuntimeError(f"Langfuse API failed {response.status_code} {path}: {response.text}")
+        return response.json()
+
+    async def _fetch_all_pages(
+        self,
+        client: httpx.AsyncClient,
+        path: str,
+        params: dict[str, Any],
+        keys: tuple[str, ...],
+        creds: LangfuseCredentials,
+    ) -> list[dict[str, Any]]:
+        page = 1
+        out: list[dict[str, Any]] = []
+        previous_signature: tuple[str, ...] | None = None
+        while True:
+            page_params = {**params, "page": page, "limit": self.PAGE_SIZE}
+            payload = await self._get_page(client, path, page_params, creds)
+            batch = _items(payload, keys)
+            if not batch:
+                break
+
+            signature = tuple(str(x.get("id") or x.get("traceId") or x.get("sessionId") or "") for x in batch[:5])
+            if page > 1 and signature == previous_signature:
+                # Protect against a legacy server ignoring the page parameter.
+                break
+            previous_signature = signature
+            out.extend(batch)
+
+            meta = payload.get("meta") if isinstance(payload, dict) and isinstance(payload.get("meta"), dict) else {}
+            total_pages = meta.get("totalPages") or meta.get("total_pages")
+            if total_pages is not None and page >= int(total_pages):
+                break
+            total_items = meta.get("totalItems") or meta.get("total_items")
+            if total_items is not None and len(out) >= int(total_items):
+                break
+            if len(batch) < self.PAGE_SIZE:
+                break
+            page += 1
+        return out
+
+    async def _fetch_sessions(self, client: httpx.AsyncClient, start: datetime, end: datetime, creds: LangfuseCredentials) -> list[dict[str, Any]]:
+        params = {"fromTimestamp": _iso_z(start), "toTimestamp": _iso_z(end)}
+        return await self._fetch_all_pages(client, "/api/public/sessions", params, ("sessions",), creds)
+
+    async def _fetch_traces(self, client: httpx.AsyncClient, start: datetime, end: datetime, creds: LangfuseCredentials) -> list[dict[str, Any]]:
+        params = {"fromTimestamp": _iso_z(start), "toTimestamp": _iso_z(end)}
+        return await self._fetch_all_pages(client, "/api/public/traces", params, ("traces",), creds)
+
+    async def _fetch_observations(self, client: httpx.AsyncClient, start: datetime, end: datetime, creds: LangfuseCredentials) -> list[dict[str, Any]]:
+        # Legacy v3 deployments differ in accepted time-filter names. Try the two
+        # known public-API conventions, but never fall back to one request per trace.
+        candidates = [
+            {"fromStartTime": _iso_z(start), "toStartTime": _iso_z(end)},
+            {"fromTimestamp": _iso_z(start), "toTimestamp": _iso_z(end)},
+        ]
+        last_error: Exception | None = None
+        for params in candidates:
+            try:
+                return await self._fetch_all_pages(client, "/api/public/observations", params, ("observations",), creds)
+            except Exception as exc:
+                last_error = exc
+        # Traces themselves normally carry input/output. Observations enrich context;
+        # a legacy deployment that cannot bulk-filter observations must not regress to N+1.
+        if last_error:
+            return []
+        return []
+
+    @staticmethod
+    def _sample_session_ids(session_ids: list[str], percentage: float, seed: str) -> list[str]:
+        unique = sorted(set(x for x in session_ids if x))
+        if not unique:
+            return []
+        if percentage >= 1:
+            return unique
+        if percentage <= 0:
+            return []
+        count = max(1, math.ceil(len(unique) * percentage))
+        selected = random.Random(seed).sample(unique, min(count, len(unique)))
+        return sorted(selected)
+
+    async def collect_sampled(
+        self,
+        period_start: datetime,
+        period_end: datetime,
+        *,
+        agent_aliases: set[str] | None,
+        percentage: float,
+        seed: str,
+        limit: int | None = None,
+    ) -> tuple[list[ConversationRecord], list[str], dict[str, Any]]:
+        if not settings.enable_langfuse:
+            raise RuntimeError("Langfuse disabled. Set ENABLE_LANGFUSE=true.")
+        creds = resolve_langfuse_credentials(self.agent_id)
+        aliases = {a for a in (agent_aliases or set()) if a}
+
+        async with httpx.AsyncClient(base_url=creds.host, timeout=90) as client:
+            sessions = await self._fetch_sessions(client, period_start, period_end, creds)
+            session_map = {sid: s for s in sessions if (sid := _session_entity_id(s))}
+            native_session_ids = set(session_map)
+
+            # Bulk-read traces and observations once for the full day. Observations
+            # are needed before sampling because historical/legacy traces may carry
+            # the real session id only in metadata instead of native trace.sessionId.
+            # This remains O(pages), never O(traces): no per-trace API calls.
+            traces = await self._fetch_traces(client, period_start, period_end, creds)
+            observations = await self._fetch_observations(client, period_start, period_end, creds)
+            obs_by_trace: dict[str, list[dict[str, Any]]] = {}
+            for obs in observations:
+                tid = obs.get("traceId") or obs.get("trace_id")
+                if tid:
+                    obs_by_trace.setdefault(str(tid), []).append(obs)
+
+            use_alias_filter = bool(aliases)
+            eligible_session_ids: set[str] = set()
+            trace_session_ids: dict[str, str] = {}
+            recovered_session_ids: set[str] = set()
+            traces_without_session = 0
+
+            for trace in traces:
+                if str(trace.get("name") or "").startswith("judge-job-"):
+                    continue
+                aid = _agent_id(trace)
+                if use_alias_filter and (not aid or aid not in aliases):
+                    continue
+
+                tid = str(trace.get("id") or "")
+                trace_observations = obs_by_trace.get(tid, [])
+                sid = _trace_session_id(trace, trace_observations)
+                if not sid:
+                    traces_without_session += 1
+                    continue
+
+                trace_session_ids[tid] = sid
+                eligible_session_ids.add(sid)
+                if sid not in session_map:
+                    session_map[sid] = _recovered_session(trace, trace_observations, sid)
+                    recovered_session_ids.add(sid)
+
+            sampled_ids = self._sample_session_ids(sorted(eligible_session_ids), percentage, seed)
+            sampled_set = set(sampled_ids)
+            selected_traces: list[dict[str, Any]] = []
+            for trace in traces:
+                if str(trace.get("name") or "").startswith("judge-job-"):
+                    continue
+                aid = _agent_id(trace)
+                if use_alias_filter and (not aid or aid not in aliases):
+                    continue
+                tid = str(trace.get("id") or "")
+                sid = trace_session_ids.get(tid)
+                if not sid or sid not in sampled_set:
+                    continue
+                selected_traces.append(trace)
+
+            selected_traces.sort(key=_trace_sort_key)
+            if limit is not None:
+                selected_traces = selected_traces[: max(0, limit)]
+
+            records: list[ConversationRecord] = []
+            for trace in selected_traces:
+                tid = str(trace.get("id") or "")
+                sid = trace_session_ids.get(tid) or ""
+                record = self._to_record(
+                    trace,
+                    obs_by_trace.get(tid, []),
+                    session_map.get(sid, {}),
+                    _agent_id(trace),
+                    resolved_session_id=sid,
+                )
+                if record.input_text or record.output_text or record.messages:
+                    records.append(record)
+
+        stats = {
+            "credential_source": creds.source,
+            "sessions_read": len(session_map),
+            "native_sessions_read": len(native_session_ids),
+            "metadata_sessions_recovered": len(recovered_session_ids),
+            "traces_without_session": traces_without_session,
+            "sampled_sessions": len(sampled_ids),
+            "traces_read": len(traces),
+            "selected_traces": len(selected_traces),
+            "observations_read": len(observations),
+        }
+        return records, sampled_ids, stats
+
+    async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None) -> list[ConversationRecord]:
+        # Generic/manual command: keep all sessions, still using the same full-day
+        # paginated path and no N+1 behavior.
+        records, _, _ = await self.collect_sampled(
+            period_start,
+            period_end,
+            agent_aliases=agent_aliases,
+            percentage=1.0,
+            seed="manual",
+            limit=limit,
+        )
+        return records
+
+    def _to_record(
+        self,
+        trace: dict[str, Any],
+        observations: list[dict[str, Any]],
+        session: dict[str, Any],
+        agent_id: str | None,
+        resolved_session_id: str | None = None,
+    ) -> ConversationRecord:
+        trace_id = str(trace.get("id") or "")
+        session_id = (
+            resolved_session_id
+            or _trace_session_id(trace, observations)
+            or _session_entity_id(session)
+            or ""
+        )
+        session_meta = _metadata(session)
+        trace_meta = _metadata(trace)
+        observations = sorted(observations, key=_observation_sort_key)
+
+        input_value = trace.get("input")
+        output_value = trace.get("output")
+        messages = _messages_from_value(input_value, "user") + _messages_from_value(output_value, "assistant")
+        for obs in observations:
+            messages.extend(_messages_from_value(obs.get("input"), "user"))
+            messages.extend(_messages_from_value(obs.get("output"), "assistant"))
+
+        # Stable de-duplication while preserving chronological order.
+        deduped: list[ConversationMessage] = []
+        seen: set[tuple[str, str]] = set()
+        for msg in messages:
+            key = (msg.role, msg.content)
+            if msg.content and key not in seen:
+                seen.add(key)
+                deduped.append(msg)
+        messages = deduped
+
+        input_text = _content_to_text(input_value).strip()
+        output_text = _content_to_text(output_value).strip()
+        if not input_text:
+            input_text = next((m.content for m in messages if m.role.lower() in {"user", "human"}), "")
+        if not output_text:
+            output_text = next((m.content for m in reversed(messages) if m.role.lower() in {"assistant", "agent", "ai"}), "")
+
+        trace_input = input_value if isinstance(input_value, dict) else {}
+        session_bc = session_meta.get("business_context") if isinstance(session_meta.get("business_context"), dict) else {}
+        trace_bc = trace_input.get("business_context") if isinstance(trace_input.get("business_context"), dict) else {}
+        identity_payload = {
+            **session_meta,
+            **trace_meta,
+            "business_context": {**session_bc, **trace_bc},
+            "session_id": session_id,
+            "message_id": _first_value(trace_meta.get("message_id"), session_meta.get("message_id")),
+            "conversation_key": _first_value(trace_meta.get("conversation_key"), session_meta.get("conversation_key")),
+        }
+        business_context = self.identity_resolver.resolve(identity_payload)
+
+        # Requirement: trace metadata first, session metadata as fallback.
+        ura_call_id = _first_value(
+            trace_meta.get("ura_call_id"), trace_meta.get("uraCallId"),
+            session_meta.get("ura_call_id"), session_meta.get("uraCallId"),
+            business_context.get("interaction_key"),
+        )
+        channel_id = _first_value(
+            trace_meta.get("channel_id"), trace_meta.get("channelId"), trace_meta.get("channel"),
+            session_meta.get("channel_id"), session_meta.get("channelId"), session_meta.get("channel"),
+        )
+        message_id = _first_value(
+            trace_meta.get("message_id"), trace_meta.get("messageId"),
+            session_meta.get("message_id"), session_meta.get("messageId"),
+            business_context.get("interaction_key"), trace_id,
+        )
+
+        metadata = {
+            **session_meta,
+            **trace_meta,
+            "business_context": business_context,
+            "ura_call_id": ura_call_id,
+            "channel_id": channel_id,
+            "session_metadata": session_meta,
+        }
+        raw = {
+            "trace": trace,
+            "session": session,
+            "observations": observations,
+            "trace_input_raw": input_value,
+            "prompt_length": _raw_character_count(input_value),
+        }
+        return ConversationRecord(
+            trace_id=trace_id,
+            # Preserve the resolved Langfuse/conversation session id. Business-context
+            # session_key may be a shortened/raw key and must not replace sessionId.
+            session_id=session_id,
+            message_id=str(message_id) if message_id is not None else None,
+            agent_id=agent_id,
+            channel=str(channel_id or "web"),
+            input_text=input_text,
+            output_text=output_text,
+            messages=messages,
+            metadata=metadata,
+            raw=raw,
+        )

evaluator/collectors/langfuse_credentials.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from evaluator.config.settings import settings
+
+
+@dataclass(frozen=True)
+class LangfuseCredentials:
+    host: str
+    public_key: str
+    secret_key: str
+    source: str
+
+    @property
+    def auth(self) -> tuple[str, str]:
+        return self.public_key, self.secret_key
+
+
+def _read_secret_file(path: str | None) -> str | None:
+    if not path:
+        return None
+    p = Path(path).expanduser()
+    if not p.is_absolute():
+        p = settings.path(p)
+    if not p.exists():
+        return None
+    value = p.read_text(encoding="utf-8").strip()
+    return value or None
+
+
+def _resolve_value(entry: dict[str, Any], direct_key: str, env_key: str, file_key: str) -> str | None:
+    """Resolve a credential field without imposing any agent-specific naming convention.
+
+    Resolution order is intentionally declarative:
+      1. direct value in the credentials definition file
+      2. environment variable whose *name* is declared in that file
+      3. file path whose *path* is declared in that file
+
+    This lets Kubernetes mount the complete credentials YAML as a Secret, or lets
+    teams keep only references to environment variables/secret files in Git.
+    """
+    direct = entry.get(direct_key)
+    if direct is not None and str(direct).strip():
+        return str(direct).strip()
+
+    env_name = entry.get(env_key)
+    if env_name:
+        value = os.getenv(str(env_name))
+        if value and value.strip():
+            return value.strip()
+
+    file_name = entry.get(file_key)
+    return _read_secret_file(str(file_name)) if file_name else None
+
+
+def load_langfuse_credential_definitions(path: str | None = None) -> dict[str, LangfuseCredentials]:
+    """Load all configured Langfuse projects and index them by logical agentId.
+
+    No agent names are hardcoded here. The function loops over whatever entries
+    are present in the definition file.
+    """
+    config_path = settings.path(path or settings.langfuse_credentials_config_path)
+    if not config_path.exists():
+        raise RuntimeError(
+            f"Langfuse credentials definition file not found: {config_path}. "
+            "Set LANGFUSE_CREDENTIALS_CONFIG_PATH to a mounted YAML file."
+        )
+
+    data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
+    root = data.get("langfuse", data)
+    entries = root.get("agents", []) if isinstance(root, dict) else []
+
+    # Also accept mapping form for operational convenience:
+    # agents:\n  my-agent: {host: ..., publicKey: ..., secretKey: ...}
+    if isinstance(entries, dict):
+        normalized = []
+        for agent_id, item in entries.items():
+            item = dict(item or {})
+            item.setdefault("agentId", agent_id)
+            normalized.append(item)
+        entries = normalized
+
+    resolved: dict[str, LangfuseCredentials] = {}
+    for item in entries or []:
+        if not isinstance(item, dict):
+            continue
+        agent_id = item.get("agentId", item.get("agent_id"))
+        if not agent_id:
+            raise ValueError("Each Langfuse credential entry must define agentId or agent_id")
+        agent_id = str(agent_id)
+
+        host = _resolve_value(item, "host", "hostEnv", "hostFile") or settings.langfuse_host
+        public_key = _resolve_value(item, "publicKey", "publicKeyEnv", "publicKeyFile")
+        secret_key = _resolve_value(item, "secretKey", "secretKeyEnv", "secretKeyFile")
+
+        if not public_key or not secret_key:
+            raise RuntimeError(
+                f"Incomplete Langfuse credentials for agent {agent_id!r} in {config_path}. "
+                "Configure publicKey/secretKey directly or their Env/File references."
+            )
+
+        resolved[agent_id] = LangfuseCredentials(
+            host=str(host).rstrip("/"),
+            public_key=public_key,
+            secret_key=secret_key,
+            source=f"{config_path}#{agent_id}",
+        )
+
+    return resolved
+
+
+def resolve_langfuse_credentials(
+    agent_id: str | None = None,
+    *,
+    allow_global_fallback: bool | None = None,
+) -> LangfuseCredentials:
+    """Resolve credentials from a declarative file, never from hardcoded agent names."""
+    if allow_global_fallback is None:
+        allow_global_fallback = agent_id is None
+
+    if agent_id:
+        definitions = load_langfuse_credential_definitions()
+        creds = definitions.get(agent_id)
+        if creds:
+            return creds
+        configured = ", ".join(sorted(definitions)) or "<none>"
+        raise RuntimeError(
+            f"No Langfuse credentials configured for agent {agent_id!r}. "
+            f"Configured agents: {configured}."
+        )
+
+    if allow_global_fallback and settings.langfuse_public_key and settings.langfuse_secret_key:
+        return LangfuseCredentials(
+            settings.langfuse_host.rstrip("/"),
+            settings.langfuse_public_key,
+            settings.langfuse_secret_key,
+            "LANGFUSE_* (manual/global fallback)",
+        )
+
+    raise RuntimeError(
+        "Langfuse credentials missing. For run-agents configure the declarative credentials file; "
+        "for the generic manual run, LANGFUSE_PUBLIC_KEY/LANGFUSE_SECRET_KEY may be used."
+    )

evaluator/collectors/mock.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+from datetime import datetime
+from evaluator.collectors.base import ConversationCollector
+from evaluator.core.models import ConversationRecord, ConversationMessage
+
+class MockCollector(ConversationCollector):
+    async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None=None, limit: int | None=None):
+        agent = next(iter(agent_aliases), 'mock-agent') if agent_aliases else 'mock-agent'
+        return [ConversationRecord(trace_id='mock-trace-1', session_id='mock-session-1', message_id='mock-message-1', agent_id=agent, channel='web', input_text='quero minha fatura', output_text='Sua fatura está em aberto no valor de R$ 120.', messages=[ConversationMessage(role='user', content='quero minha fatura'), ConversationMessage(role='assistant', content='Sua fatura está em aberto no valor de R$ 120.')], metadata={'mock': True})]

evaluator/config/agents.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+import yaml
+from pathlib import Path
+from evaluator.config.settings import settings
+
+@dataclass
+class AgentConfig:
+    agent_id: str
+    enabled: bool = True
+    days_back: int = 1
+    percentage: float = 1.0
+    langfuse_agent_aliases: list[str] = field(default_factory=list)
+    gcs_prefix: str = ""
+
+    @property
+    def aliases(self) -> set[str]:
+        return {self.agent_id, *self.langfuse_agent_aliases}
+
+
+def load_agents(path: str | None = None) -> list[AgentConfig]:
+    p = settings.path(path or settings.agents_config_path)
+    data = yaml.safe_load(p.read_text()) or {}
+    agents = []
+    for item in data.get("agents", []):
+        agent_id = item.get("agent_id", item.get("agentId"))
+        if not agent_id:
+            raise ValueError("Each agent entry must define agentId or agent_id")
+        cfg = AgentConfig(
+            agent_id=str(agent_id),
+            enabled=bool(item.get("enabled", True)),
+            days_back=int(item.get("days_back", item.get("daysBack", 1))),
+            percentage=float(item.get("percentage", 1.0)),
+            langfuse_agent_aliases=list(item.get("langfuse_agent_aliases", [])),
+            gcs_prefix=str(item.get("gcs_prefix", "")),
+        )
+        if cfg.enabled:
+            agents.append(cfg)
+    return agents

evaluator/config/settings.py

@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from dotenv import load_dotenv
+
+ROOT_DIR = Path(__file__).resolve().parents[2]
+ENV_PATH = ROOT_DIR / ".env"
+load_dotenv(ENV_PATH, override=True)
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=str(ENV_PATH), extra="ignore", case_sensitive=False)
+
+    app_env: str = Field(default="dev", validation_alias="APP_ENV")
+
+    adb_user: str = Field(default="", validation_alias="ADB_USER")
+    adb_password: str = Field(default="", validation_alias="ADB_PASSWORD")
+    adb_dsn: str = Field(default="", validation_alias="ADB_DSN")
+    adb_wallet_location: str | None = Field(default=None, validation_alias="ADB_WALLET_LOCATION")
+    adb_wallet_password: str | None = Field(default=None, validation_alias="ADB_WALLET_PASSWORD")
+    adb_table_prefix: str = Field(default="AGENTFW", validation_alias="ADB_TABLE_PREFIX")
+
+    enable_langfuse: bool = Field(default=False, validation_alias="ENABLE_LANGFUSE")
+    langfuse_public_key: str | None = Field(default=None, validation_alias="LANGFUSE_PUBLIC_KEY")
+    langfuse_secret_key: str | None = Field(default=None, validation_alias="LANGFUSE_SECRET_KEY")
+    langfuse_host: str = Field(default="http://localhost:3005", validation_alias="LANGFUSE_HOST")
+    publish_langfuse_scores: bool = Field(default=False, validation_alias="PUBLISH_LANGFUSE_SCORES")
+    langfuse_credentials_config_path: str = Field(
+        default="configs/langfuse/langfuse_credentials.yaml",
+        validation_alias="LANGFUSE_CREDENTIALS_CONFIG_PATH",
+    )
+
+    # llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER")
+    # llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE")
+    # llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH")
+    # oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT")
+    # oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID")
+    # oci_genai_compartment_id: str | None = Field(default=None, validation_alias="OCI_GENAI_COMPARTMENT_ID")
+    # oci_genai_auth_type: str = Field(default="api_key", validation_alias="OCI_GENAI_AUTH_TYPE")
+    # oci_config_path: str | None = Field(default=None, validation_alias="OCI_CONFIG_PATH")
+    # oci_config_profile: str = Field(default="DEFAULT", validation_alias="OCI_CONFIG_PROFILE")
+    # llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE")
+    # llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS")
+
+    # LLM / OCI GenAI OpenAI-compatible, mesmo padrão do Agent Framework
+    llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER")
+    llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE")
+    llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH")
+
+    oci_genai_base_url: str | None = Field(default=None, validation_alias="OCI_GENAI_BASE_URL")
+    oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT")  # compatibilidade
+    oci_genai_model: str = Field(default="openai.gpt-4.1", validation_alias="OCI_GENAI_MODEL")
+    oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID")  # compatibilidade
+    oci_genai_api_key: str | None = Field(default=None, validation_alias="OCI_GENAI_API_KEY")
+    oci_genai_project_ocid: str | None = Field(default=None, validation_alias="OCI_GENAI_PROJECT_OCID")
+    oci_genai_endpoint_id: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT_ID")
+    oci_auth_mode: str = Field(default="config_file", validation_alias="OCI_AUTH_MODE")
+    oci_config_file: str = Field(default="~/.oci/config", validation_alias="OCI_CONFIG_FILE")
+    oci_profile: str = Field(default="DEFAULT", validation_alias="OCI_PROFILE")
+    oci_compartment_id: str | None = Field(default=None, validation_alias="OCI_COMPARTMENT_ID")
+    oci_region: str = Field(default="sa-saopaulo-1", validation_alias="OCI_REGION")
+
+    llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE")
+    llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS")
+    llm_timeout_seconds: int = Field(default=120, validation_alias="LLM_TIMEOUT_SECONDS")
+
+    agents_config_path: str = Field(default="configs/judge/agents.yaml", validation_alias="AGENTS_CONFIG_PATH")
+    trace_prompt_path: str = Field(default="configs/judge/trace_metrics.yaml", validation_alias="TRACE_PROMPT_PATH")
+    session_prompt_path: str = Field(default="configs/judge/session_metrics.yaml", validation_alias="SESSION_PROMPT_PATH")
+    output_dir: str = Field(default="output", validation_alias="OUTPUT_DIR")
+    batch_size: int = Field(default=50, validation_alias="BATCH_SIZE")
+    max_attempts: int = Field(default=3, validation_alias="MAX_ATTEMPTS")
+    context_turn_limit: int = Field(default=6, validation_alias="CONTEXT_TURN_LIMIT")
+    enable_gcs_upload: bool = Field(default=False, validation_alias="ENABLE_GCS_UPLOAD")
+    judge_gcs_bucket: str | None = Field(default=None, validation_alias="JUDGE_GCS_BUCKET")
+    storage_config_path: str = Field(default="configs/judge/storage.yaml", validation_alias="STORAGE_CONFIG_PATH")
+    google_application_credentials: str | None = Field(default=None, validation_alias="GOOGLE_APPLICATION_CREDENTIALS")
+    identity_config_path: str = "configs/identity.yaml"
+
+    @property
+    def project_root(self) -> Path:
+        return ROOT_DIR
+
+    def path(self, value: str | Path) -> Path:
+        p = Path(value)
+        return p if p.is_absolute() else ROOT_DIR / p
+
+    @property
+    def ADB_USER(self): return self.adb_user
+    @property
+    def ADB_PASSWORD(self): return self.adb_password
+    @property
+    def ADB_DSN(self): return self.adb_dsn
+    @property
+    def ADB_WALLET_LOCATION(self): return self.adb_wallet_location
+    @property
+    def ADB_WALLET_PASSWORD(self): return self.adb_wallet_password
+    @property
+    def ADB_TABLE_PREFIX(self): return (self.adb_table_prefix or "AGENTFW").upper().rstrip("_")
+
+    @property
+    def has_langfuse_credentials(self) -> bool:
+        return bool(self.langfuse_public_key and self.langfuse_secret_key)
+
+    @property
+    def can_use_langfuse(self) -> bool:
+        return bool(self.enable_langfuse and self.has_langfuse_credentials)
+
+    @property
+    def can_publish_langfuse_scores(self) -> bool:
+        return bool(self.publish_langfuse_scores and self.can_use_langfuse)
+
+    @property
+    def OCI_GENAI_BASE_URL(self) -> str | None:
+        return self.oci_genai_base_url or self.oci_genai_endpoint
+
+    @property
+    def OCI_GENAI_MODEL(self) -> str:
+        return self.oci_genai_model_id or self.oci_genai_model
+
+    @property
+    def OCI_GENAI_API_KEY(self) -> str | None:
+        return self.oci_genai_api_key
+
+    @property
+    def OCI_GENAI_PROJECT_OCID(self) -> str | None:
+        return self.oci_genai_project_ocid
+
+    @property
+    def LLM_PROVIDER(self) -> str:
+        return self.llm_provider
+
+    @property
+    def LLM_TEMPERATURE(self) -> float:
+        return self.llm_temperature
+
+    @property
+    def LLM_MAX_TOKENS(self) -> int:
+        return self.llm_max_tokens
+
+    @property
+    def LLM_TIMEOUT_SECONDS(self) -> int:
+        return self.llm_timeout_seconds
+
+    @property
+    def LLM_PROFILES_PATH(self) -> str:
+        return self.llm_profiles_path
+
+settings = Settings()
+if settings.ADB_WALLET_LOCATION:
+    os.environ["TNS_ADMIN"] = settings.ADB_WALLET_LOCATION

evaluator/core/models.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+from datetime import datetime
+from enum import Enum
+from pydantic import BaseModel, Field
+from typing import Any
+
+class RunStatus(str, Enum):
+    RUNNING = "RUNNING"
+    COMPLETED = "COMPLETED"
+    PARTIAL = "PARTIAL"
+    FAILED = "FAILED"
+
+class ItemStatus(str, Enum):
+    PENDING = "PENDING"
+    PROCESSING = "PROCESSING"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
+    SKIPPED = "SKIPPED"
+
+class ConversationMessage(BaseModel):
+    role: str
+    content: str
+    created_at: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+class ConversationRecord(BaseModel):
+    trace_id: str | None = None
+    session_id: str
+    message_id: str | None = None
+    agent_id: str | None = None
+    channel: str | None = None
+    input_text: str = ""
+    output_text: str = ""
+    messages: list[ConversationMessage] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    raw: dict[str, Any] = Field(default_factory=dict)
+
+class TraceJudgeResult(BaseModel):
+    judgeScore: float | None = Field(default=None, ge=0.0, le=1.0)
+    accuracyScore: float | None = Field(default=None, ge=0.0, le=1.0)
+    alucinationScore: float | None = Field(default=None, ge=0.0, le=1.0)
+    rationale: str = ""
+    judge_name: str = "trace_metrics"
+    judge_type: str = "trace"
+
+class SessionJudgeResult(BaseModel):
+    inferredCsiScore: float = Field(ge=0.0, le=1.0)
+    resolution: int = Field(ge=0, le=1)
+    conversationPrecision: int = Field(ge=0, le=1)
+    rationale: str = ""
+    judge_name: str = "session_metrics"
+    judge_type: str = "session"
+
+class CombinedJudgeResult(BaseModel):
+    trace: TraceJudgeResult
+    session: SessionJudgeResult | None = None

evaluator/engine.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+import asyncio, inspect, json, math, random
+from datetime import date, datetime
+from typing import Any, Awaitable, Callable
+from evaluator.collectors.base import ConversationCollector
+from evaluator.collectors.langfuse import LangfuseCollector
+from evaluator.collectors.agent_framework import AgentFrameworkCollector
+from evaluator.collectors.mock import MockCollector
+from evaluator.config.agents import AgentConfig
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationRecord, RunStatus
+from evaluator.judges.llm_judge import TIMStyleLLMJudge
+from evaluator.output.legacy_exporter import export_legacy_txt_gz
+from evaluator.persistence.repository import EvaluationRepository
+from evaluator.publishers.langfuse_scores import LangfuseScorePublisher
+from evaluator.publishers.langfuse_job_audit import LangfuseJobAuditPublisher
+from evaluator.publishers.gcs_export import upload_evaluation_file
+
+ProgressCallback = Callable[[dict[str, Any]], Awaitable[None] | None]
+
+
+class EvaluationEngine:
+    def __init__(self, repository: EvaluationRepository | None = None, progress_callback: ProgressCallback | None = None):
+        self.repository = repository or EvaluationRepository(auto_init_schema=False)
+        self.progress_callback = progress_callback
+        self.judge = TIMStyleLLMJudge()
+        self.langfuse_publisher = LangfuseScorePublisher()
+        self.job_audit = LangfuseJobAuditPublisher()
+
+    async def _emit(self, progress_run_id: str, stage: str, message: str = "", **details):
+        details.pop("run_id", None)
+        await self.repository.arecord_progress(progress_run_id, stage, message, details)
+        event = {"run_id": progress_run_id, "stage": stage, "message": message, "details": details}
+        if self.progress_callback:
+            r = self.progress_callback(event)
+            if inspect.isawaitable(r):
+                await r
+
+    def collector_for(self, source: str, agent_id: str | None = None) -> ConversationCollector:
+        if source == 'langfuse': return LangfuseCollector(agent_id=agent_id)
+        if source == 'agent_framework': return AgentFrameworkCollector()
+        if source == 'mock': return MockCollector()
+        raise ValueError('source must be langfuse, agent_framework or mock')
+
+    async def _audit(self, progress_run_id: str, audit_trace_id: str, agent_id: str, status: str, period_start: datetime, period_end: datetime, *, required: bool = True, **details) -> bool:
+        try:
+            result = await self.job_audit.publish(
+                audit_trace_id, agent_id, status, period_start=period_start, period_end=period_end, details=details
+            )
+            if not result or not result.get("ok"):
+                raise RuntimeError(f"Langfuse audit did not confirm persistence: {result!r}")
+            await self._emit(progress_run_id, 'LANGFUSE_JOB_AUDIT', f'judge-job-{agent_id} status={status}', audit_status=status, audit_result=result)
+            return True
+        except Exception as exc:
+            await self._emit(progress_run_id, 'LANGFUSE_JOB_AUDIT_FAILED', f'Langfuse job audit failed: {exc}', audit_status=status, error=str(exc))
+            if required:
+                raise
+            return False
+
+    async def run_agent(self, agent: AgentConfig, period_start: datetime, period_end: datetime, source: str = 'langfuse', limit: int | None = None, reference_date: date | None = None) -> dict:
+        run_id = await self.repository.acreate_run(period_start, period_end, source, agent.agent_id)
+        reference_date = reference_date or period_start.date()
+        audit_trace_id = self.job_audit.new_trace_id()
+        try:
+            # Emit RUN_CREATED immediately after persistence so CLI/monitor can expose
+            # the RUN_ID before any external Langfuse audit/network call is performed.
+            await self._emit(run_id, 'RUN_CREATED', f'Agent run created: {agent.agent_id}', agent_id=agent.agent_id, source=source, reference_date=reference_date.isoformat())
+            await self._audit(run_id, audit_trace_id, agent.agent_id, 'created', period_start, period_end, run_id=run_id, source=source)
+            collector = self.collector_for(source, agent.agent_id)
+            await self._emit(run_id, 'COLLECTING', 'Collecting sessions and conversations')
+            seed = f'{agent.agent_id}:{reference_date.isoformat()}'
+            if isinstance(collector, LangfuseCollector):
+                records, sampled_sessions, collection_stats = await collector.collect_sampled(
+                    period_start, period_end, agent_aliases=agent.aliases, percentage=agent.percentage, seed=seed, limit=limit
+                )
+                await self._emit(
+                    run_id, 'COLLECTED', f'Collected {len(records)} traces from {len(sampled_sessions)} sampled sessions',
+                    **collection_stats
+                )
+            else:
+                records = await collector.collect(period_start, period_end, agent_aliases=agent.aliases, limit=limit)
+                records, sampled_sessions = self._sample_sessions(records, agent.percentage, agent.agent_id, reference_date)
+                collection_stats = {}
+                await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} traces before session sampling')
+            await self._emit(
+                run_id,
+                'SAMPLED',
+                f'Kept {len(sampled_sessions)} sessions / {len(records)} traces',
+                percentage=agent.percentage,
+                seed=seed,
+                sampled_sessions=sampled_sessions,
+            )
+            inserted = await self.repository.ainsert_items(run_id, records)
+            await self._emit(run_id, 'ITEMS_INSERTED', f'Inserted {inserted} items')
+            summary = await self._process(run_id)
+            output_path = export_legacy_txt_gz(self.repository, run_id, agent.agent_id)
+            await self._emit(run_id, 'EXPORTED', f'Exported {output_path}', output_file=str(output_path))
+            if not settings.enable_gcs_upload:
+                raise RuntimeError("TIM specification requires GCS upload for run-agents; set ENABLE_GCS_UPLOAD=true")
+            uploaded_to = await asyncio.to_thread(upload_evaluation_file, output_path, agent.gcs_prefix)
+            await self._emit(run_id, 'GCS_UPLOADED', f'Uploaded {uploaded_to}', uploaded_to=uploaded_to)
+            output_path.unlink()
+            await self._emit(run_id, 'LOCAL_FILE_DELETED', 'Local export deleted after successful GCS upload')
+            result = {**summary, 'agent_id': agent.agent_id, 'output_file': str(output_path), 'uploaded_to': uploaded_to}
+            # _audit() receives status as an explicit positional argument. The run
+            # summary also contains a "status" key (e.g. COMPLETED), so forwarding
+            # **result unchanged would pass status twice and raise:
+            #   TypeError: _audit() got multiple values for argument 'status'
+            audit_details = dict(result)
+            # Never forward names that are explicit _audit() parameters.
+            for reserved_key in ('progress_run_id', 'audit_trace_id', 'agent_id', 'status', 'period_start', 'period_end'):
+                audit_details.pop(reserved_key, None)
+            await self._audit(run_id, audit_trace_id, agent.agent_id, 'completed', period_start, period_end, **audit_details)
+            return result
+        except Exception as exc:
+            await self.repository.amark_run_status(run_id, RunStatus.FAILED, str(exc))
+            await self._emit(run_id, 'FAILED', f'Agent run failed: {exc}', error=str(exc))
+            await self._audit(run_id, audit_trace_id, agent.agent_id, 'failed', period_start, period_end, required=False, run_id=run_id, error=str(exc))
+            return {'status': 'FAILED', 'run_id': run_id, 'agent_id': agent.agent_id, 'error': str(exc)}
+
+    async def run(self, period_start: datetime, period_end: datetime, source: str = 'langfuse', limit: int | None = None) -> dict:
+        run_id = await self.repository.acreate_run(period_start, period_end, source, None)
+        try:
+            collector = self.collector_for(source)
+            await self._emit(run_id, 'COLLECTING', 'Collecting conversations')
+            records = await collector.collect(period_start, period_end, limit=limit)
+            await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} records')
+            await self.repository.ainsert_items(run_id, records)
+            return await self._process(run_id)
+        except Exception as exc:
+            await self.repository.amark_run_status(run_id, RunStatus.PARTIAL, str(exc))
+            await self._emit(run_id, 'PARTIAL', f'Run failed: {exc}', error=str(exc))
+            return {'status': 'PARTIAL', 'run_id': run_id, 'error': str(exc)}
+
+    async def _process(self, run_id: str) -> dict:
+        processed_records: list[ConversationRecord] = []
+        while True:
+            items = await self.repository.afetch_next_items(run_id, settings.batch_size)
+            if not items: break
+            await self._emit(run_id, 'BATCH_STARTED', f'Processing {len(items)} items')
+            for item in items:
+                item_id = item['item_id']
+                await self.repository.amark_item_processing(item_id)
+                try:
+                    raw = item['raw_json']
+                    if hasattr(raw, 'read'): raw = raw.read()
+                    record = ConversationRecord.model_validate(json.loads(raw))
+                    result = await self.judge.judge_trace(record)
+                    await self.repository.asave_trace_result(run_id, item_id, record, result)
+                    await self.langfuse_publisher.publish_trace_score(record, result)
+                    await self.repository.amark_item_completed(run_id, item_id)
+                    processed_records.append(record)
+                    await self._emit(
+                        run_id,
+                        "ITEM_COMPLETED",
+                        f"Item completed {item_id}",
+                        trace_id=record.trace_id,
+                        session_id=record.session_id,
+                        judgeScore=result.judgeScore,
+                        accuracyScore=result.accuracyScore,
+                        alucinationScore=result.alucinationScore,
+                        rationale=result.rationale,
+                    )
+                except Exception as exc:
+                    await self.repository.amark_item_failed(run_id, item_id, str(exc))
+                    await self._emit(run_id, 'ITEM_FAILED', f'Item failed {item_id}', error=str(exc))
+        if processed_records:
+            sessions = await self.judge.judge_sessions(processed_records)
+            for sid, result in sessions.items():
+                agent_id = next((r.agent_id for r in processed_records if r.session_id == sid), None)
+                await self.repository.asave_session_result(run_id, sid, agent_id, result)
+            await self._emit(run_id, 'SESSION_JUDGE_COMPLETED', f'Evaluated {len(sessions)} sessions')
+        await self.repository.amark_run_status(run_id, RunStatus.COMPLETED)
+        summary = await self.repository.asummarize_run(run_id)
+        await self._emit(run_id, 'COMPLETED', 'Run completed', **summary)
+        return {'status': 'COMPLETED', **summary}
+
+    def _sample_sessions(self, records: list[ConversationRecord], percentage: float, agent_id: str, reference_date: date) -> tuple[list[ConversationRecord], list[str]]:
+        """Deterministic session sampling; all traces from selected sessions are retained."""
+        if not records:
+            return [], []
+        session_ids = sorted({r.session_id for r in records})
+        if percentage >= 1:
+            selected = session_ids
+        elif percentage <= 0:
+            selected = []
+        else:
+            count = min(len(session_ids), max(1, math.ceil(len(session_ids) * percentage)))
+            rng = random.Random(f"{agent_id}:{reference_date.isoformat()}")
+            selected = sorted(rng.sample(session_ids, count))
+        selected_set = set(selected)
+        return [r for r in records if r.session_id in selected_set], selected

evaluator/identity/resolver.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _deep_get(data: dict, path: str):
+    cur = data
+    for part in path.split("."):
+        if not isinstance(cur, dict):
+            return None
+        cur = cur.get(part)
+    return cur
+
+
+class IdentityResolver:
+    def __init__(self, path: str = "configs/identity.yaml"):
+        self.path = Path(path)
+        self.config = yaml.safe_load(self.path.read_text(encoding="utf-8")) or {}
+        self.identity = self.config.get("identity", {})
+        self.keys = self.identity.get("keys", {})
+
+    def resolve(self, payload: dict[str, Any]) -> dict[str, Any]:
+        out = {}
+
+        for key, spec in self.keys.items():
+            value = None
+            for source in spec.get("sources", []):
+                value = _deep_get(payload, source)
+                if value not in (None, ""):
+                    break
+            out[key] = str(value) if value not in (None, "") else None
+
+        out["metadata"] = {
+            "identity_version": self.identity.get("version"),
+            "identity_source": str(self.path),
+        }
+
+        return out

evaluator/judges/llm_judge.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+import json
+import re
+from collections import defaultdict
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationRecord, TraceJudgeResult, SessionJudgeResult
+from evaluator.llm.client import LLMClient, create_llm_client
+from evaluator.prompts.loader import load_prompt
+
+
+def _json_from_text(text: str) -> dict:
+    try:
+        return json.loads(text)
+    except Exception:
+        m = re.search(r"\{.*\}", text, flags=re.S)
+        if not m:
+            raise
+        return json.loads(m.group(0))
+
+
+def _history(record: ConversationRecord, turn_limit: int | None = None, max_chars: int = 6000) -> str:
+    messages = record.messages or []
+    if messages:
+        if turn_limit and turn_limit > 0:
+            # A turn is represented by one conversational message in the normalized record.
+            messages = messages[-turn_limit:]
+        text = "\n".join(f"{m.role}: {m.content}" for m in messages)
+    else:
+        text = f"user: {record.input_text}\nagent: {record.output_text}"
+    return text[-max_chars:]
+
+
+class TIMStyleLLMJudge:
+    def __init__(self, llm: LLMClient | None = None):
+        self.llm = llm or create_llm_client()
+        self.trace_prompt = load_prompt(settings.trace_prompt_path, 'trace_metrics')
+        self.session_prompt = load_prompt(settings.session_prompt_path, 'session_metrics')
+
+    @staticmethod
+    def _validated_trace_data(raw: str) -> dict:
+        data = _json_from_text(raw)
+        normalized = dict(data)
+        normalized.setdefault("judgeScore", normalized.get("judge_score"))
+        normalized.setdefault("accuracyScore", normalized.get("accuracy_score"))
+        normalized.setdefault("alucinationScore", normalized.get("alucination_score"))
+        normalized.setdefault("rationale", normalized.get("reasoning", ""))
+        # Pydantic validates the 0..1 contract here. Invalid metric output is treated
+        # like an invalid judge payload and receives the single specification retry.
+        TraceJudgeResult(**normalized)
+        return normalized
+
+    async def _trace_json_with_retry(self, prompt: str) -> tuple[dict | None, str | None]:
+        """One normal attempt + one retry-hint for invalid JSON/metric contract."""
+        raw = await self.llm.complete(prompt)
+        try:
+            return self._validated_trace_data(raw), None
+        except Exception as first_exc:
+            retry_prompt = f"""{prompt}
+
+RETRY-HINT:
+A resposta anterior não pôde ser interpretada como JSON válido ou violou o contrato das métricas.
+Retorne SOMENTE um objeto JSON válido, sem markdown, comentários ou texto adicional,
+com judgeScore, accuracyScore e alucinationScore entre 0.0 e 1.0 e a chave rationale.
+"""
+            retry_raw = await self.llm.complete(retry_prompt)
+            try:
+                return self._validated_trace_data(retry_raw), None
+            except Exception as second_exc:
+                return None, f"parse_failed_after_retry: {second_exc or first_exc}"
+
+    async def judge_trace(self, record: ConversationRecord) -> TraceJudgeResult:
+        prompt = f"""{self.trace_prompt}
+
+HISTÓRICO:
+{_history(record, turn_limit=settings.context_turn_limit)}
+
+MENSAGEM DO USUÁRIO:
+{record.input_text}
+
+RESPOSTA DO AGENTE:
+{record.output_text}
+
+METADATA:
+{json.dumps(record.metadata, ensure_ascii=False, default=str)}
+"""
+        data, parse_error = await self._trace_json_with_retry(prompt)
+        if data is None:
+            # Specification: after the second parse failure metrics become None;
+            # legacy exporter serializes those values as "0.00".
+            return TraceJudgeResult(
+                judgeScore=None,
+                accuracyScore=None,
+                alucinationScore=None,
+                rationale=parse_error or "parse_failed_after_retry",
+            )
+
+        data.setdefault("judge_name", "trace_metrics")
+        data.setdefault("judge_type", "trace")
+        data.setdefault("judgeScore", data.get("judge_score"))
+        data.setdefault("accuracyScore", data.get("accuracy_score"))
+        data.setdefault("alucinationScore", data.get("alucination_score"))
+        data.setdefault("rationale", data.get("reasoning", ""))
+        return TraceJudgeResult(**data)
+
+    async def judge_sessions(self, records: list[ConversationRecord]) -> dict[str, SessionJudgeResult]:
+        grouped: dict[str, list[ConversationRecord]] = defaultdict(list)
+        for r in records:
+            grouped[r.session_id].append(r)
+        out = {}
+        for session_id, items in grouped.items():
+            transcript = "\n".join(_history(r, max_chars=3000) for r in items)[-9000:]
+            prompt = f"""{self.session_prompt}
+
+TRANSCRIÇÃO DA SESSÃO:
+{transcript}
+"""
+            raw = await self.llm.complete(prompt)
+            data = _json_from_text(raw)
+            data.setdefault("judge_name", "session_metrics")
+            data.setdefault("judge_type", "session")
+            data.setdefault("inferredCsiScore", data.get("inferred_csi_score", 0))
+            data.setdefault("resolution", data.get("resolution", 0))
+            data.setdefault("conversationPrecision", data.get("conversation_precision", 0))
+            data.setdefault("rationale", data.get("reasoning", ""))
+            out[session_id] = SessionJudgeResult(**data)
+        return out

evaluator/llm/client.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Any
+
+from evaluator.config.settings import settings
+from evaluator.llm.profile_resolver import LLMProfileResolver
+
+
+class LLMClient:
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        raise NotImplementedError
+
+
+class MockLLMClient(LLMClient):
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        if "inferredCsiScore" in prompt:
+            return json.dumps({
+                "inferredCsiScore": 0.5,
+                "resolution": 1,
+                "conversationPrecision": 1,
+                "rationale": "Avaliação mock."
+            }, ensure_ascii=False)
+        return json.dumps({
+            "judgeScore": 0.7,
+            "accuracyScore": 0.7,
+            "alucinationScore": 0.1,
+            "rationale": "Avaliação mock."
+        }, ensure_ascii=False)
+
+
+class OCICompatibleLLMClient(LLMClient):
+    def __init__(self):
+        self.resolver = LLMProfileResolver(settings)
+
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        effective = self.resolver.resolve(profile_name or settings.llm_profile)
+        provider = str(effective.get("provider") or settings.LLM_PROVIDER).lower()
+        model = str(effective.get("model") or settings.OCI_GENAI_MODEL)
+        base_url = effective.get("base_url") or settings.OCI_GENAI_BASE_URL
+        api_key = effective.get("api_key") or settings.OCI_GENAI_API_KEY
+        temperature = effective.get("temperature", settings.LLM_TEMPERATURE)
+        max_tokens = effective.get("max_tokens", settings.LLM_MAX_TOKENS)
+        timeout = effective.get("timeout_seconds", settings.LLM_TIMEOUT_SECONDS)
+
+        if provider == "mock":
+            return await MockLLMClient().complete(prompt, profile_name=profile_name)
+        if provider not in ("oci_openai", "openai_compatible"):
+            raise ValueError(f"Unsupported LLM provider: {provider}")
+        if not base_url:
+            raise RuntimeError("OCI_GENAI_BASE_URL is required for oci_openai provider")
+        if not api_key:
+            raise RuntimeError("OCI_GENAI_API_KEY is required for oci_openai provider")
+
+        from openai import AsyncOpenAI
+        client = AsyncOpenAI(base_url=base_url, api_key=api_key, timeout=timeout)
+        resp = await client.chat.completions.create(
+            model=model,
+            messages=[{"role": "user", "content": prompt}],
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+        return resp.choices[0].message.content or ""
+
+
+class OCISDKLLMClient(LLMClient):
+    """OCI Generative AI native SDK, including dedicated endpoints."""
+
+    def __init__(self):
+        self.resolver = LLMProfileResolver(settings)
+        self._clients: dict[str, Any] = {}
+
+    @staticmethod
+    def _normalize_endpoint(value: str | None) -> str:
+        endpoint = (value or "").strip().strip("'\"").rstrip("/")
+        if not endpoint:
+            region = settings.oci_region or "sa-saopaulo-1"
+            return f"https://inference.generativeai.{region}.oci.oraclecloud.com"
+        if "/openai/v1" in endpoint:
+            raise RuntimeError(
+                "OCI_GENAI_BASE_URL cannot contain /openai/v1 when LLM_PROVIDER=oci_sdk"
+            )
+        return endpoint
+
+    @staticmethod
+    def _config_and_signer():
+        import oci
+
+        mode = (settings.oci_auth_mode or "config_file").lower()
+        if mode == "config_file":
+            config_file = str(Path(settings.oci_config_file).expanduser())
+            return oci.config.from_file(config_file, settings.oci_profile), None
+        if mode == "instance_principal":
+            return {}, oci.auth.signers.InstancePrincipalsSecurityTokenSigner()
+        if mode == "resource_principal":
+            return {}, oci.auth.signers.get_resource_principals_signer()
+        if mode in ("oke_workload_identity", "workload_identity"):
+            return {}, oci.auth.signers.get_oke_workload_identity_resource_principal_signer()
+        raise ValueError(f"Unsupported OCI_AUTH_MODE={settings.oci_auth_mode}")
+
+    def _client(self, endpoint: str):
+        if endpoint not in self._clients:
+            from oci.generative_ai_inference import GenerativeAiInferenceClient
+            config, signer = self._config_and_signer()
+            kwargs: dict[str, Any] = {"config": config, "service_endpoint": endpoint}
+            if signer is not None:
+                kwargs["signer"] = signer
+            self._clients[endpoint] = GenerativeAiInferenceClient(**kwargs)
+        return self._clients[endpoint]
+
+    @staticmethod
+    def _extract_answer(response: Any) -> str:
+        data = getattr(response, "data", response)
+        chat_response = getattr(data, "chat_response", None) or data
+        for attr in ("text", "message", "output_text"):
+            value = getattr(chat_response, attr, None)
+            if value:
+                return str(value)
+        choices = getattr(chat_response, "choices", None) or []
+        if choices:
+            first = choices[0]
+            msg = getattr(first, "message", None)
+            content = getattr(msg, "content", None) if msg is not None else getattr(first, "text", None)
+            if isinstance(content, list):
+                return "".join(str(getattr(item, "text", item)) for item in content)
+            if content:
+                return str(content)
+        return str(chat_response)
+
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        from oci.generative_ai_inference import models
+
+        effective = self.resolver.resolve(profile_name or settings.llm_profile)
+        model = str(effective.get("model") or settings.OCI_GENAI_MODEL).strip()
+        endpoint_id = str(effective.get("endpoint_id") or settings.oci_genai_endpoint_id or "").strip() or None
+        if not endpoint_id and model.startswith("ocid1.generativeaiendpoint."):
+            endpoint_id = model
+        compartment_id = (
+            effective.get("compartment_id")
+            or settings.oci_compartment_id
+            or settings.OCI_GENAI_PROJECT_OCID
+        )
+        if not compartment_id:
+            raise RuntimeError(
+                "OCI_COMPARTMENT_ID or OCI_GENAI_PROJECT_OCID is required for LLM_PROVIDER=oci_sdk"
+            )
+
+        endpoint = self._normalize_endpoint(effective.get("base_url") or settings.OCI_GENAI_BASE_URL)
+        if endpoint_id:
+            serving_mode = models.DedicatedServingMode(endpoint_id=endpoint_id)
+        else:
+            serving_mode = models.OnDemandServingMode(model_id=model)
+
+        content = [models.TextContent(text=prompt)] if hasattr(models, "TextContent") else prompt
+        message = models.UserMessage(content=content)
+        chat_request = models.GenericChatRequest(
+            messages=[message],
+            temperature=effective.get("temperature", settings.LLM_TEMPERATURE),
+            max_tokens=effective.get("max_tokens", settings.LLM_MAX_TOKENS),
+        )
+        details = models.ChatDetails(
+            compartment_id=compartment_id,
+            serving_mode=serving_mode,
+            chat_request=chat_request,
+        )
+        response = await asyncio.to_thread(self._client(endpoint).chat, details)
+        return self._extract_answer(response)
+
+
+def create_llm_client() -> LLMClient:
+    provider = (settings.LLM_PROVIDER or "mock").lower()
+    if provider in ("mock", "none"):
+        return MockLLMClient()
+    if provider in ("oci_openai", "openai_compatible", "oci"):
+        return OCICompatibleLLMClient()
+    if provider == "oci_sdk":
+        return OCISDKLLMClient()
+    raise ValueError(f"Unsupported LLM_PROVIDER={settings.LLM_PROVIDER}")

evaluator/llm/profile_resolver.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import copy
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _canonical_profile_name(value: str | None) -> str:
+    name = (value or "default").strip()
+    name = name.replace("-", "_").replace(".", "_").replace(" ", "_")
+    name = re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
+    return re.sub(r"_+", "_", name).strip("_") or "default"
+
+
+class LLMProfileResolver:
+    def __init__(self, settings: Any):
+        self.settings = settings
+        self.path = self._find_profiles_file()
+        self.enabled = self.path is not None
+        self._profiles = self._load_profiles(self.path) if self.enabled else {}
+
+    def _find_profiles_file(self) -> Path | None:
+        root = getattr(self.settings, "project_root", Path.cwd())
+        configured = Path(getattr(self.settings, "LLM_PROFILES_PATH", "") or "").expanduser()
+        candidates = []
+        if configured:
+            candidates.append(configured if configured.is_absolute() else Path(root) / configured)
+        candidates += [
+            Path(root) / "llm_profiles.yaml",
+            Path(root) / "configs/llm_profiles/llm_profiles.yaml",
+            Path(root) / "config/llm_profiles.yaml",
+            Path("llm_profiles.yaml"),
+            Path("configs/llm_profiles/llm_profiles.yaml"),
+        ]
+        for path in candidates:
+            if path and path.exists() and path.is_file():
+                return path
+        return None
+
+    def _load_profiles(self, path: Path) -> dict[str, dict[str, Any]]:
+        # Expand ${VAR} placeholders, matching the way env-driven framework config is commonly used.
+        text = os.path.expandvars(path.read_text(encoding="utf-8"))
+        data = yaml.safe_load(text) or {}
+        raw = data.get("profiles", data)
+        profiles = {}
+        for name, value in raw.items():
+            if isinstance(value, dict):
+                profiles[_canonical_profile_name(str(name))] = dict(value)
+        return profiles
+
+    def env_defaults(self) -> dict[str, Any]:
+        return {
+            "provider": getattr(self.settings, "LLM_PROVIDER", "mock"),
+            "model": getattr(self.settings, "OCI_GENAI_MODEL", "mock-llm"),
+            "temperature": getattr(self.settings, "LLM_TEMPERATURE", 0.0),
+            "max_tokens": getattr(self.settings, "LLM_MAX_TOKENS", 900),
+            "timeout_seconds": getattr(self.settings, "LLM_TIMEOUT_SECONDS", 120),
+            "base_url": getattr(self.settings, "OCI_GENAI_BASE_URL", None),
+            "api_key": getattr(self.settings, "OCI_GENAI_API_KEY", None),
+            "project_ocid": getattr(self.settings, "OCI_GENAI_PROJECT_OCID", None),
+            "endpoint_id": getattr(self.settings, "oci_genai_endpoint_id", None),
+            "compartment_id": getattr(self.settings, "oci_compartment_id", None),
+        }
+
+    def resolve(self, profile_name: str | None = None, **overrides) -> dict[str, Any]:
+        profile_key = _canonical_profile_name(profile_name)
+        effective = self.env_defaults()
+
+        if self.enabled:
+            effective.update(copy.deepcopy(self._profiles.get("default") or {}))
+            effective.update(copy.deepcopy(self._profiles.get(profile_key) or {}))
+
+        for key, value in overrides.items():
+            if value is not None:
+                effective[key] = value
+
+        effective["profile_name"] = profile_key
+        return effective

evaluator/output/legacy_exporter.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import gzip
+import json
+from collections import defaultdict
+from pathlib import Path
+from datetime import datetime
+from zoneinfo import ZoneInfo
+from typing import Any
+
+from evaluator.config.settings import settings
+from evaluator.persistence.repository import EvaluationRepository
+from evaluator.analytics.vloop import max_consecutive_intent_streak
+
+HEADER = [
+    "judgeScore", "accuracyScore", "alucinationScore", "promptLength", "loop",
+    "inferredCsiScore", "resolution", "conversationPrecision",
+    "uraCallId", "channelId", "sessionId", "messageId"
+]
+
+
+def _q(v) -> str:
+    return '"' + str("" if v is None else v).replace('"', '""') + '"'
+
+
+def _metric(v) -> str | float | int:
+    # TIM specification: trace metric None must be serialized as "0.00".
+    return "0.00" if v is None else v
+
+
+def normalize_raw(raw):
+    if hasattr(raw, "read"):
+        raw = raw.read()
+    if isinstance(raw, bytes):
+        raw = raw.decode("utf-8")
+    if isinstance(raw, str):
+        raw = raw.strip()
+        if not raw:
+            return {}
+        raw = json.loads(raw)
+    if isinstance(raw, str):
+        raw = json.loads(raw)
+    return raw if isinstance(raw, dict) else {}
+
+
+def extract_prompt_length(raw: dict) -> int:
+    """TIM metric: raw character count of trace.input captured by the collector."""
+    nested = raw.get("raw") if isinstance(raw.get("raw"), dict) else {}
+    if "prompt_length" in nested:
+        try:
+            return int(nested.get("prompt_length") or 0)
+        except Exception:
+            pass
+    return len(str(raw.get("input_text") or ""))
+
+
+def extract_ura_call_id(raw: dict, metadata: dict | None = None, message_id: str | None = None) -> str:
+    metadata = metadata or {}
+    raw_metadata = raw.get("metadata") if isinstance(raw.get("metadata"), dict) else {}
+    business_context = (
+        metadata.get("business_context")
+        or metadata.get("businessContext")
+        or raw.get("business_context")
+        or raw.get("businessContext")
+        or raw_metadata.get("business_context")
+        or raw_metadata.get("businessContext")
+        or {}
+    )
+    if not isinstance(business_context, dict):
+        business_context = {}
+
+    nested = raw.get("raw") if isinstance(raw.get("raw"), dict) else {}
+    trace = nested.get("trace") if isinstance(nested.get("trace"), dict) else {}
+    detail = nested.get("detail") if isinstance(nested.get("detail"), dict) else {}
+    trace_input = trace.get("input") if isinstance(trace.get("input"), dict) else {}
+    detail_input = detail.get("input") if isinstance(detail.get("input"), dict) else {}
+    trace_metadata = trace.get("metadata") if isinstance(trace.get("metadata"), dict) else {}
+    detail_metadata = detail.get("metadata") if isinstance(detail.get("metadata"), dict) else {}
+    trace_bc = trace_input.get("business_context") if isinstance(trace_input.get("business_context"), dict) else {}
+    detail_bc = detail_input.get("business_context") if isinstance(detail_input.get("business_context"), dict) else {}
+
+    return str(
+        business_context.get("interaction_key")
+        or business_context.get("ura_call_id")
+        or metadata.get("ura_call_id")
+        or metadata.get("uraCallId")
+        or metadata.get("interaction_key")
+        or trace_metadata.get("ura_call_id")
+        or detail_metadata.get("ura_call_id")
+        or trace_bc.get("interaction_key")
+        or detail_bc.get("interaction_key")
+        or message_id
+        or ""
+    )
+
+
+def _session_loop_map(rows) -> dict[str, int]:
+    grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
+    for row in rows:
+        session_id = row[1]
+        raw_json = row[8]
+        try:
+            grouped[str(session_id)].append(normalize_raw(raw_json))
+        except Exception:
+            grouped[str(session_id)].append({})
+    return {sid: max_consecutive_intent_streak(items) for sid, items in grouped.items()}
+
+
+def export_legacy_txt_gz(repo: EvaluationRepository, run_id: str, agent_id: str) -> Path:
+    output_dir = settings.path(settings.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    now_brt = datetime.now(ZoneInfo("America/Sao_Paulo"))
+    path = output_dir / f"AGENTE_{agent_id}_LLM_JUDGE_{now_brt.strftime('%Y%m%d_%H%M%S')}.TXT.GZ"
+
+    with repo.store.connect() as conn:
+        cur = conn.cursor()
+        cur.execute(f"""
+            select SESSION_ID, INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION
+              from {repo.store.t('EVALUATION_RESULT')}
+             where RUN_ID = :run_id and JUDGE_TYPE = 'SESSION'
+        """, {"run_id": run_id})
+        session_metrics = {
+            sid: {"inferredCsiScore": csi, "resolution": res, "conversationPrecision": prec}
+            for sid, csi, res, prec in cur.fetchall()
+        }
+
+        cur.execute(f"""
+            select r.TRACE_ID, r.SESSION_ID, r.JUDGE_SCORE, r.ACCURACY_SCORE,
+                   r.ALUCINATION_SCORE, r.RATIONALE, i.CHANNEL, i.MESSAGE_ID, i.RAW_JSON
+              from {repo.store.t('EVALUATION_RESULT')} r
+              left join {repo.store.t('EVALUATION_ITEM')} i on i.ITEM_ID = r.ITEM_ID
+             where r.RUN_ID = :run_id and r.JUDGE_TYPE = 'TRACE'
+             order by r.CREATED_AT
+        """, {"run_id": run_id})
+        rows = cur.fetchall()
+
+    loop_by_session = _session_loop_map(rows)
+
+    with gzip.open(path, "wt", encoding="utf-8") as f:
+        f.write("|;".join(_q(v) for v in HEADER) + "\n")
+        for trace_id, session_id, judge, accuracy, alucination, rationale, channel, message_id, raw_json in rows:
+            session = session_metrics.get(session_id, {})
+            raw = {}
+            ura_call_id = ""
+            channel_id = channel or ""
+            prompt_length = 0
+            try:
+                raw = normalize_raw(raw_json)
+                metadata = raw.get("metadata") if isinstance(raw.get("metadata"), dict) else {}
+                channel_id = (
+                    metadata.get("channel_id")
+                    or metadata.get("channelId")
+                    or metadata.get("channel")
+                    or channel_id
+                )
+                ura_call_id = extract_ura_call_id(raw, metadata, message_id)
+                prompt_length = extract_prompt_length(raw)
+            except Exception as exc:
+                print(f"[legacy_exporter] metadata extraction failed trace_id={trace_id}: {exc}")
+
+            vals = [
+                _metric(judge),
+                _metric(accuracy),
+                _metric(alucination),
+                prompt_length,
+                loop_by_session.get(str(session_id), 0),
+                session.get("inferredCsiScore"),
+                session.get("resolution"),
+                session.get("conversationPrecision"),
+                ura_call_id,
+                channel_id,
+                session_id,
+                message_id or trace_id,
+            ]
+            f.write("|;".join(_q(v) for v in vals) + "\n")
+        f.write("|;".join([_q("TOTAL"), _q(len(rows))]) + "\n")
+    return path

evaluator/persistence/oracle_store.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+
+def _json_dumps(value: Any) -> str:
+    return json.dumps(value or {}, ensure_ascii=False, default=str)
+
+
+def _json_loads(value: str | bytes | None, default: Any):
+    if value is None:
+        return default
+    if isinstance(value, bytes):
+        value = value.decode("utf-8")
+    try:
+        return json.loads(value)
+    except Exception:
+        return default
+
+
+@dataclass
+class OracleSettings:
+    user: str
+    password: str
+    dsn: str
+    wallet_location: str | None = None
+    wallet_password: str | None = None
+    table_prefix: str = "AGENTFW"
+
+
+class OracleStore:
+    """Oracle Autonomous Database store following the Agent Framework pattern.
+
+    Uses direct oracledb.connect() with wallet arguments. Synchronous DB calls are
+    exposed through asyncio.to_thread(), matching the main framework style.
+    """
+
+    def __init__(self, settings, auto_init_schema: bool = False):
+        self.settings = settings
+        self.cfg = OracleSettings(
+            user=settings.ADB_USER or "",
+            password=settings.ADB_PASSWORD or "",
+            dsn=settings.ADB_DSN or "",
+            wallet_location=getattr(settings, "ADB_WALLET_LOCATION", None),
+            wallet_password=getattr(settings, "ADB_WALLET_PASSWORD", None),
+            table_prefix=(getattr(settings, "ADB_TABLE_PREFIX", "AGENTFW") or "AGENTFW").upper().rstrip("_"),
+        )
+        if not self.cfg.user or not self.cfg.password or not self.cfg.dsn:
+            raise RuntimeError("ADB_USER, ADB_PASSWORD and ADB_DSN are required")
+        if auto_init_schema:
+            self._init_schema()
+
+    @staticmethod
+    def now() -> datetime:
+        return datetime.now(timezone.utc)
+
+    def t(self, name: str) -> str:
+        return f"{self.cfg.table_prefix}_{name}".upper()
+
+    @contextmanager
+    def connect(self):
+        import oracledb
+        oracledb.defaults.fetch_lobs = False
+        kwargs = {}
+        if self.cfg.wallet_location:
+            kwargs["config_dir"] = self.cfg.wallet_location
+            kwargs["wallet_location"] = self.cfg.wallet_location
+        if self.cfg.wallet_password:
+            kwargs["wallet_password"] = self.cfg.wallet_password
+        conn = oracledb.connect(user=self.cfg.user, password=self.cfg.password, dsn=self.cfg.dsn, **kwargs)
+        try:
+            yield conn
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+        finally:
+            conn.close()
+
+    async def to_thread(self, func, *args, **kwargs):
+        return await asyncio.to_thread(func, *args, **kwargs)
+
+    def _exec_ddl_ignore_exists(self, cur, ddl: str):
+        try:
+            cur.execute(ddl)
+        except Exception as exc:
+            msg = str(exc)
+            if "ORA-00955" in msg or "ORA-01408" in msg or "ORA-02275" in msg:
+                return
+            raise
+
+    def _column_exists(self, cur, table_name: str, column_name: str) -> bool:
+        cur.execute("""
+            select count(*)
+              from user_tab_columns
+             where table_name = :table_name
+               and column_name = :column_name
+        """, {"table_name": self.t(table_name), "column_name": column_name.upper()})
+        return int(cur.fetchone()[0] or 0) > 0
+
+    def _ensure_column(self, cur, table_name: str, column_name: str, ddl_type: str):
+        if not self._column_exists(cur, table_name, column_name):
+            cur.execute(f"alter table {self.t(table_name)} add {column_name.upper()} {ddl_type}")
+
+    def drop_schema(self):
+        tables = [
+            "EVALUATION_RESULT",
+            "EVALUATION_FINDING",
+            "EVALUATION_METRIC",
+            "EVALUATION_ITEM",
+            "EVALUATION_PROGRESS_EVENT",
+            "EVALUATION_RUN",
+        ]
+        with self.connect() as conn:
+            cur = conn.cursor()
+            for table in tables:
+                try:
+                    cur.execute(f"drop table {self.t(table)} cascade constraints purge")
+                except Exception as exc:
+                    if "ORA-00942" in str(exc):
+                        continue
+                    raise
+
+    def _init_schema(self):
+        with self.connect() as conn:
+            cur = conn.cursor()
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_RUN')} (
+                    RUN_ID varchar2(64) primary key,
+                    AGENT_ID varchar2(128),
+                    SOURCE varchar2(64),
+                    PERIOD_START timestamp with time zone,
+                    PERIOD_END timestamp with time zone,
+                    STATUS varchar2(32) not null,
+                    TOTAL_ITEMS number default 0 not null,
+                    PROCESSED_ITEMS number default 0 not null,
+                    FAILED_ITEMS number default 0 not null,
+                    RETRY_COUNT number default 0 not null,
+                    ERROR_MESSAGE clob,
+                    LAST_HEARTBEAT_AT timestamp with time zone,
+                    CREATED_AT timestamp with time zone not null,
+                    UPDATED_AT timestamp with time zone not null
+                )
+            """)
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_PROGRESS_EVENT')} (
+                    ID number generated always as identity primary key,
+                    RUN_ID varchar2(64) not null,
+                    STAGE varchar2(128) not null,
+                    MESSAGE varchar2(1000),
+                    DETAILS_JSON clob check (DETAILS_JSON is json),
+                    CREATED_AT timestamp with time zone not null,
+                    constraint {self.t('FK_EVAL_PROGRESS_RUN')}
+                        foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID)
+                )
+            """)
+            self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_PROGRESS_RUN')} on {self.t('EVALUATION_PROGRESS_EVENT')}(RUN_ID, CREATED_AT)")
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_ITEM')} (
+                    ITEM_ID varchar2(64) primary key,
+                    RUN_ID varchar2(64) not null,
+                    TRACE_ID varchar2(256),
+                    SESSION_ID varchar2(256),
+                    MESSAGE_ID varchar2(256),
+                    AGENT_ID varchar2(128),
+                    CHANNEL varchar2(64),
+                    STATUS varchar2(32) not null,
+                    ATTEMPT_COUNT number default 0 not null,
+                    ERROR_MESSAGE clob,
+                    RAW_JSON clob check (RAW_JSON is json),
+                    CREATED_AT timestamp with time zone not null,
+                    UPDATED_AT timestamp with time zone not null,
+                    constraint {self.t('FK_EVAL_ITEM_RUN')}
+                        foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID)
+                )
+            """)
+            self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_ITEM_RUN')} on {self.t('EVALUATION_ITEM')}(RUN_ID, STATUS, CREATED_AT)")
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_RESULT')} (
+                    RESULT_ID varchar2(64) primary key,
+                    RUN_ID varchar2(64) not null,
+                    ITEM_ID varchar2(64),
+                    TRACE_ID varchar2(256),
+                    SESSION_ID varchar2(256),
+                    AGENT_ID varchar2(128),
+                    JUDGE_NAME varchar2(128) not null,
+                    JUDGE_TYPE varchar2(32),
+                    SCORE number,
+                    JUDGE_SCORE number,
+                    ACCURACY_SCORE number,
+                    ALUCINATION_SCORE number,
+                    HALLUCINATION_SCORE number,
+                    INFERRED_CSI_SCORE number,
+                    RESOLUTION number,
+                    CONVERSATION_PRECISION number,
+                    TOOL_USAGE_SCORE number,
+                    ROUTING_SCORE number,
+                    RATIONALE clob,
+                    REASONING clob,
+                    RESULT_JSON clob check (RESULT_JSON is json),
+                    CREATED_AT timestamp with time zone not null,
+                    constraint {self.t('FK_EVAL_RESULT_RUN')}
+                        foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID),
+                    constraint {self.t('FK_EVAL_RESULT_ITEM')}
+                        foreign key (ITEM_ID) references {self.t('EVALUATION_ITEM')}(ITEM_ID)
+                )
+            """)
+            self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_RESULT_RUN')} on {self.t('EVALUATION_RESULT')}(RUN_ID, ITEM_ID)")
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_METRIC')} (
+                    METRIC_ID varchar2(64) primary key,
+                    RUN_ID varchar2(64) not null,
+                    ITEM_ID varchar2(64),
+                    METRIC_NAME varchar2(128) not null,
+                    METRIC_VALUE number,
+                    DIMENSIONS_JSON clob check (DIMENSIONS_JSON is json),
+                    CREATED_AT timestamp with time zone not null,
+                    constraint {self.t('FK_EVAL_METRIC_RUN')}
+                        foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID)
+                )
+            """)
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_FINDING')} (
+                    FINDING_ID varchar2(64) primary key,
+                    RUN_ID varchar2(64) not null,
+                    ITEM_ID varchar2(64),
+                    SEVERITY varchar2(32),
+                    CATEGORY varchar2(128),
+                    TITLE varchar2(512),
+                    DESCRIPTION clob,
+                    EVIDENCE_JSON clob check (EVIDENCE_JSON is json),
+                    CREATED_AT timestamp with time zone not null,
+                    constraint {self.t('FK_EVAL_FINDING_RUN')}
+                        foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID)
+                )
+            """)
+
+            # Non-destructive compatibility for older generated schemas.
+            for col, typ in [
+                ("RETRY_COUNT", "number default 0"),
+                ("ERROR_MESSAGE", "clob"),
+                ("LAST_HEARTBEAT_AT", "timestamp with time zone"),
+                ("UPDATED_AT", "timestamp with time zone"),
+            ]:
+                self._ensure_column(cur, "EVALUATION_RUN", col, typ)
+            for col, typ in [
+                ("ID", "number generated always as identity"),
+            ]:
+                # Identity column cannot always be added cleanly; ignore if table is old without ID.
+                try:
+                    self._ensure_column(cur, "EVALUATION_PROGRESS_EVENT", col, typ)
+                except Exception:
+                    pass
+            for col, typ in [
+                ("JUDGE_NAME", "varchar2(128) default 'unknown_judge' not null"),
+                ("JUDGE_TYPE", "varchar2(32)"),
+                ("SCORE", "number"),
+                ("JUDGE_SCORE", "number"),
+                ("ACCURACY_SCORE", "number"),
+                ("ALUCINATION_SCORE", "number"),
+                ("HALLUCINATION_SCORE", "number"),
+                ("INFERRED_CSI_SCORE", "number"),
+                ("RESOLUTION", "number"),
+                ("CONVERSATION_PRECISION", "number"),
+                ("TOOL_USAGE_SCORE", "number"),
+                ("ROUTING_SCORE", "number"),
+                ("RATIONALE", "clob"),
+                ("REASONING", "clob"),
+                ("RESULT_JSON", "clob"),
+            ]:
+                self._ensure_column(cur, "EVALUATION_RESULT", col, typ)

evaluator/persistence/repository.py

@@ -0,0 +1,411 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationRecord, ItemStatus, RunStatus, TraceJudgeResult, SessionJudgeResult
+from evaluator.persistence.oracle_store import OracleStore, _json_dumps, _json_loads
+from evaluator.time_utils import brt_naive
+
+
+class EvaluationRepository:
+    def __init__(self, auto_init_schema: bool = False):
+        self.store = OracleStore(settings, auto_init_schema=auto_init_schema)
+
+    def create_run(self, period_start: datetime, period_end: datetime, source: str, agent_id: str | None = None) -> str:
+        run_id = str(uuid.uuid4())
+        now = self.store.now()
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                insert into {self.store.t('EVALUATION_RUN')}
+                (RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS, TOTAL_ITEMS,
+                 PROCESSED_ITEMS, FAILED_ITEMS, RETRY_COUNT, LAST_HEARTBEAT_AT, CREATED_AT, UPDATED_AT)
+                values (:run_id, :agent_id, :period_start, :period_end, :source, :status,
+                        0, 0, 0, 0, :heartbeat_at, :created_at, :updated_at)
+            """, {
+                "run_id": run_id,
+                "agent_id": agent_id,
+                "period_start": brt_naive(period_start),
+                "period_end": brt_naive(period_end),
+                "source": source,
+                "status": RunStatus.RUNNING.value,
+                "heartbeat_at": now,
+                "created_at": now,
+                "updated_at": now,
+            })
+        return run_id
+
+    async def acreate_run(self, *args, **kwargs):
+        return await self.store.to_thread(self.create_run, *args, **kwargs)
+
+    def record_progress(self, run_id: str, stage: str, message: str = "", details: dict | None = None):
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                insert into {self.store.t('EVALUATION_PROGRESS_EVENT')}
+                (RUN_ID, STAGE, MESSAGE, DETAILS_JSON, CREATED_AT)
+                values (:run_id, :stage, :message, :details_json, :created_at)
+            """, {
+                "run_id": run_id,
+                "stage": stage,
+                "message": (message or "")[:1000],
+                "details_json": _json_dumps(details or {}),
+                "created_at": self.store.now(),
+            })
+
+    async def arecord_progress(self, *args, **kwargs):
+        return await self.store.to_thread(self.record_progress, *args, **kwargs)
+
+    def insert_items(self, run_id: str, records: list[ConversationRecord]) -> int:
+        inserted = 0
+        now = self.store.now()
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            for record in records:
+                try:
+                    cur.execute(f"""
+                        insert into {self.store.t('EVALUATION_ITEM')}
+                        (ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL,
+                         STATUS, ATTEMPT_COUNT, RAW_JSON, CREATED_AT, UPDATED_AT)
+                        values (:item_id, :run_id, :trace_id, :session_id, :message_id, :agent_id,
+                                :channel, :status, 0, :raw_json, :created_at, :updated_at)
+                    """, {
+                        "item_id": str(uuid.uuid4()),
+                        "run_id": run_id,
+                        "trace_id": record.trace_id,
+                        "session_id": record.session_id,
+                        "message_id": record.message_id,
+                        "agent_id": record.agent_id,
+                        "channel": record.channel,
+                        "status": ItemStatus.PENDING.value,
+                        "raw_json": record.model_dump_json(),
+                        "created_at": now,
+                        "updated_at": now,
+                    })
+                    inserted += 1
+                except Exception as exc:
+                    if "ORA-00001" not in str(exc):
+                        raise
+            cur.execute(f"""
+                update {self.store.t('EVALUATION_RUN')}
+                   set TOTAL_ITEMS = (
+                       select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id
+                   ),
+                   UPDATED_AT = :updated_at
+                 where RUN_ID = :run_id
+            """, {"run_id": run_id, "updated_at": self.store.now()})
+        return inserted
+
+    async def ainsert_items(self, *args, **kwargs):
+        return await self.store.to_thread(self.insert_items, *args, **kwargs)
+
+    def fetch_next_items(self, run_id: str, batch_size: int) -> list[dict]:
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                select * from (
+                    select ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL,
+                           STATUS, ATTEMPT_COUNT, RAW_JSON
+                      from {self.store.t('EVALUATION_ITEM')}
+                     where RUN_ID = :run_id
+                       and STATUS in (:pending, :failed)
+                       and ATTEMPT_COUNT < :max_attempts
+                     order by CREATED_AT
+                ) where rownum <= :batch_size
+            """, {
+                "run_id": run_id,
+                "pending": ItemStatus.PENDING.value,
+                "failed": ItemStatus.FAILED.value,
+                "max_attempts": settings.max_attempts,
+                "batch_size": batch_size,
+            })
+            cols = [d[0].lower() for d in cur.description]
+            return [dict(zip(cols, row)) for row in cur.fetchall()]
+
+    async def afetch_next_items(self, *args, **kwargs):
+        return await self.store.to_thread(self.fetch_next_items, *args, **kwargs)
+
+    def mark_item_processing(self, item_id: str):
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                update {self.store.t('EVALUATION_ITEM')}
+                   set STATUS = :status,
+                       ATTEMPT_COUNT = ATTEMPT_COUNT + 1,
+                       UPDATED_AT = :updated_at
+                 where ITEM_ID = :item_id
+            """, {
+                "status": ItemStatus.PROCESSING.value,
+                "updated_at": self.store.now(),
+                "item_id": item_id,
+            })
+
+    async def amark_item_processing(self, *args, **kwargs):
+        return await self.store.to_thread(self.mark_item_processing, *args, **kwargs)
+
+    def mark_item_completed(self, run_id: str, item_id: str):
+        now = self.store.now()
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                update {self.store.t('EVALUATION_ITEM')}
+                   set STATUS = :status,
+                       UPDATED_AT = :updated_at
+                 where ITEM_ID = :item_id
+            """, {
+                "status": ItemStatus.COMPLETED.value,
+                "updated_at": now,
+                "item_id": item_id,
+            })
+            self._refresh_run_counters(cur, run_id, now)
+
+    async def amark_item_completed(self, *args, **kwargs):
+        return await self.store.to_thread(self.mark_item_completed, *args, **kwargs)
+
+    def mark_item_failed(self, run_id: str, item_id: str, error: str):
+        now = self.store.now()
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                update {self.store.t('EVALUATION_ITEM')}
+                   set STATUS = :status,
+                       ERROR_MESSAGE = :error,
+                       UPDATED_AT = :updated_at
+                 where ITEM_ID = :item_id
+            """, {
+                "status": ItemStatus.FAILED.value,
+                "error": (error or "")[:4000],
+                "updated_at": now,
+                "item_id": item_id,
+            })
+            self._refresh_run_counters(cur, run_id, now)
+
+    async def amark_item_failed(self, *args, **kwargs):
+        return await self.store.to_thread(self.mark_item_failed, *args, **kwargs)
+
+    def _refresh_run_counters(self, cur, run_id: str, updated_at):
+        cur.execute(f"""
+            update {self.store.t('EVALUATION_RUN')}
+               set PROCESSED_ITEMS = (
+                   select count(*) from {self.store.t('EVALUATION_ITEM')}
+                    where RUN_ID = :run_id and STATUS = :completed
+               ),
+               FAILED_ITEMS = (
+                   select count(*) from {self.store.t('EVALUATION_ITEM')}
+                    where RUN_ID = :run_id and STATUS = :failed
+               ),
+               UPDATED_AT = :updated_at
+             where RUN_ID = :run_id
+        """, {
+            "run_id": run_id,
+            "completed": ItemStatus.COMPLETED.value,
+            "failed": ItemStatus.FAILED.value,
+            "updated_at": updated_at,
+        })
+
+    def save_trace_result(self, run_id: str, item_id: str, record: ConversationRecord, result: TraceJudgeResult):
+        judge_name = getattr(result, "judge_name", None) or "trace_metrics"
+        judge_type = (getattr(result, "judge_type", None) or "TRACE").upper()
+        score = getattr(result, "judgeScore", None)
+        accuracy = getattr(result, "accuracyScore", None)
+        alucination = getattr(result, "alucinationScore", None)
+        rationale = getattr(result, "rationale", None) or ""
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                insert into {self.store.t('EVALUATION_RESULT')}
+                (RESULT_ID, RUN_ID, ITEM_ID, TRACE_ID, SESSION_ID, AGENT_ID, JUDGE_NAME,
+                 JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE, ALUCINATION_SCORE,
+                 RATIONALE, RESULT_JSON, CREATED_AT)
+                values (:result_id, :run_id, :item_id, :trace_id, :session_id, :agent_id,
+                        :judge_name, :judge_type, :score, :judge_score, :accuracy_score,
+                        :alucination_score, :rationale, :result_json, :created_at)
+            """, {
+                "result_id": str(uuid.uuid4()),
+                "run_id": run_id,
+                "item_id": item_id,
+                "trace_id": record.trace_id,
+                "session_id": record.session_id,
+                "agent_id": record.agent_id,
+                "judge_name": judge_name,
+                "judge_type": judge_type,
+                "score": score,
+                "judge_score": score,
+                "accuracy_score": accuracy,
+                "alucination_score": alucination,
+                "rationale": rationale,
+                "result_json": result.model_dump_json(),
+                "created_at": self.store.now(),
+            })
+
+    async def asave_trace_result(self, *args, **kwargs):
+        return await self.store.to_thread(self.save_trace_result, *args, **kwargs)
+
+    def save_session_result(self, run_id: str, session_id: str, agent_id: str | None, result: SessionJudgeResult):
+        judge_name = getattr(result, "judge_name", None) or "session_metrics"
+        judge_type = (getattr(result, "judge_type", None) or "SESSION").upper()
+        rationale = getattr(result, "rationale", None) or ""
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                insert into {self.store.t('EVALUATION_RESULT')}
+                (RESULT_ID, RUN_ID, SESSION_ID, AGENT_ID, JUDGE_NAME, JUDGE_TYPE,
+                 INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION, RATIONALE,
+                 RESULT_JSON, CREATED_AT)
+                values (:result_id, :run_id, :session_id, :agent_id, :judge_name, :judge_type,
+                        :csi, :resolution, :precision, :rationale, :result_json, :created_at)
+            """, {
+                "result_id": str(uuid.uuid4()),
+                "run_id": run_id,
+                "session_id": session_id,
+                "agent_id": agent_id,
+                "judge_name": judge_name,
+                "judge_type": judge_type,
+                "csi": getattr(result, "inferredCsiScore", None),
+                "resolution": getattr(result, "resolution", None),
+                "precision": getattr(result, "conversationPrecision", None),
+                "rationale": rationale,
+                "result_json": result.model_dump_json(),
+                "created_at": self.store.now(),
+            })
+
+    async def asave_session_result(self, *args, **kwargs):
+        return await self.store.to_thread(self.save_session_result, *args, **kwargs)
+
+    def mark_run_status(self, run_id: str, status: RunStatus, error: str | None = None):
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                update {self.store.t('EVALUATION_RUN')}
+                   set STATUS = :status,
+                       ERROR_MESSAGE = :error,
+                       UPDATED_AT = :updated_at
+                 where RUN_ID = :run_id
+            """, {
+                "status": status.value,
+                "error": error,
+                "updated_at": self.store.now(),
+                "run_id": run_id,
+            })
+
+    async def amark_run_status(self, *args, **kwargs):
+        return await self.store.to_thread(self.mark_run_status, *args, **kwargs)
+
+    def summarize_run(self, run_id: str) -> dict:
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                select
+                    (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id),
+                    (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'COMPLETED'),
+                    (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'FAILED'),
+                    (select count(*) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE'),
+                    (select avg(JUDGE_SCORE) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE')
+                from dual
+            """, {"run_id": run_id})
+            r = cur.fetchone()
+            return {
+                "run_id": run_id,
+                "total_items": int(r[0] or 0),
+                "completed_items": int(r[1] or 0),
+                "failed_items": int(r[2] or 0),
+                "evaluations": int(r[3] or 0),
+                "avg_score": float(r[4]) if r[4] is not None else None,
+            }
+
+    async def asummarize_run(self, *args, **kwargs):
+        return await self.store.to_thread(self.summarize_run, *args, **kwargs)
+
+    def get_run_progress(self, run_id: str, event_limit: int = 20) -> dict:
+        summary = self.summarize_run(run_id)
+        total = summary["total_items"] or 0
+        done = summary["completed_items"] + summary["failed_items"]
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                select * from (
+                    select STAGE, MESSAGE, DETAILS_JSON, CREATED_AT
+                      from {self.store.t('EVALUATION_PROGRESS_EVENT')}
+                     where RUN_ID = :run_id
+                     order by CREATED_AT desc
+                ) where rownum <= :max_rows
+            """, {"run_id": run_id, "max_rows": event_limit})
+            events = [
+                {
+                    "stage": s,
+                    "message": m,
+                    "details": _json_loads(d.read() if hasattr(d, "read") else d, {}),
+                    "created_at": str(c),
+                }
+                for s, m, d, c in cur.fetchall()
+            ]
+        return {
+            **summary,
+            "done_items": done,
+            "percent_complete": round((done / total) * 100, 2) if total else 0.0,
+            "events": events,
+        }
+
+    async def aget_run_progress(self, *args, **kwargs):
+        return await self.store.to_thread(self.get_run_progress, *args, **kwargs)
+
+    def list_runs(self, limit: int = 20):
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                select * from (
+                    select RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS,
+                           TOTAL_ITEMS, PROCESSED_ITEMS, FAILED_ITEMS, CREATED_AT, UPDATED_AT
+                      from {self.store.t('EVALUATION_RUN')}
+                     order by CREATED_AT desc
+                ) where rownum <= :max_rows
+            """, {"max_rows": limit})
+            return [
+                {
+                    "run_id": r[0],
+                    "agent_id": r[1],
+                    "period_start": str(r[2]),
+                    "period_end": str(r[3]),
+                    "source": r[4],
+                    "status": r[5],
+                    "total_items": int(r[6] or 0),
+                    "processed_items": int(r[7] or 0),
+                    "failed_items": int(r[8] or 0),
+                    "created_at": str(r[9]),
+                    "updated_at": str(r[10]),
+                }
+                for r in cur.fetchall()
+            ]
+
+    async def alist_runs(self, *args, **kwargs):
+        return await self.store.to_thread(self.list_runs, *args, **kwargs)
+
+    def list_results(self, run_id: str, limit: int = 100) -> list[dict]:
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                select JUDGE_NAME, JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE,
+                       ALUCINATION_SCORE, INFERRED_CSI_SCORE, RESOLUTION,
+                       CONVERSATION_PRECISION, RATIONALE, TRACE_ID, SESSION_ID, CREATED_AT
+                  from {self.store.t('EVALUATION_RESULT')}
+                 where RUN_ID = :run_id
+                 order by CREATED_AT desc
+            """, {"run_id": run_id})
+            return [
+                {
+                    "judge_name": r[0],
+                    "judge_type": r[1],
+                    "score": r[2],
+                    "judge_score": r[3],
+                    "accuracy_score": r[4],
+                    "alucination_score": r[5],
+                    "inferred_csi_score": r[6],
+                    "resolution": r[7],
+                    "conversation_precision": r[8],
+                    "rationale": r[9],
+                    "trace_id": r[10],
+                    "session_id": r[11],
+                    "created_at": str(r[12]),
+                }
+                for r in cur.fetchall()[:limit]
+            ]
+
+    async def alist_results(self, *args, **kwargs):
+        return await self.store.to_thread(self.list_results, *args, **kwargs)

evaluator/prompts/loader.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+import yaml
+from evaluator.config.settings import settings
+
+
+def load_prompt(path: str, key: str) -> str:
+    p = settings.path(path)
+    data = yaml.safe_load(p.read_text()) or {}
+    if key not in data:
+        raise KeyError(f"Prompt key {key!r} not found in {p}")
+    return str(data[key])

evaluator/publishers/gcs_export.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from pathlib import Path
+import yaml
+
+from evaluator.config.settings import settings
+
+
+def _load_bucket_mapping() -> dict[str, str]:
+    config_path = settings.path(settings.storage_config_path)
+    if not config_path.exists():
+        raise RuntimeError(
+            f"Storage configuration file not found: {config_path}. "
+            "Set STORAGE_CONFIG_PATH to a YAML file containing storage.buckets."
+        )
+    data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
+    storage = data.get("storage", {}) if isinstance(data, dict) else {}
+    buckets = storage.get("buckets", {}) if isinstance(storage, dict) else {}
+    if not isinstance(buckets, dict):
+        raise RuntimeError(f"Invalid storage.buckets mapping in {config_path}")
+    return {str(k).lower(): str(v) for k, v in buckets.items() if v}
+
+
+def resolve_gcs_bucket() -> str:
+    if settings.judge_gcs_bucket:
+        return settings.judge_gcs_bucket
+    env = (settings.app_env or "").lower()
+    bucket = _load_bucket_mapping().get(env)
+    if not bucket:
+        raise RuntimeError(
+            f"No GCS bucket configured for APP_ENV={settings.app_env!r}. "
+            "Configure storage.buckets in STORAGE_CONFIG_PATH or set JUDGE_GCS_BUCKET."
+        )
+    return bucket
+
+
+def upload_evaluation_file(path: Path, prefix: str = "") -> str:
+    if not settings.enable_gcs_upload:
+        raise RuntimeError("GCS upload requested while ENABLE_GCS_UPLOAD=false")
+
+    from google.cloud import storage
+
+    bucket_name = resolve_gcs_bucket()
+    object_name = "/".join(
+        part.strip("/") for part in (prefix, path.name) if part and part.strip("/")
+    )
+
+    client = storage.Client()
+    bucket = client.bucket(bucket_name)
+    blob = bucket.blob(object_name)
+    blob.upload_from_filename(str(path), content_type="application/gzip")
+    return f"gs://{bucket_name}/{object_name}"

evaluator/publishers/langfuse_job_audit.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timezone
+from typing import Any
+
+import httpx
+
+from evaluator.collectors.langfuse_credentials import resolve_langfuse_credentials
+from evaluator.config.settings import settings
+
+
+class LangfuseJobAuditPublisher:
+    """Audit each agent batch as judge-job-{agentId} in that agent's Langfuse project."""
+
+    def new_trace_id(self) -> str:
+        return uuid.uuid4().hex
+
+    async def publish(
+        self,
+        trace_id: str,
+        agent_id: str,
+        status: str,
+        *,
+        period_start: datetime | None = None,
+        period_end: datetime | None = None,
+        details: dict[str, Any] | None = None,
+    ) -> dict[str, Any] | None:
+        if not settings.enable_langfuse:
+            return None
+        creds = resolve_langfuse_credentials(agent_id)
+        event_timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+        body: dict[str, Any] = {
+            "id": trace_id,
+            "timestamp": event_timestamp,
+            "name": f"judge-job-{agent_id}",
+            "metadata": {"status": status, "agentId": agent_id, **(details or {})},
+        }
+        if period_start or period_end:
+            body["input"] = {
+                "period_start": period_start.isoformat() if period_start else None,
+                "period_end": period_end.isoformat() if period_end else None,
+            }
+        if status in {"completed", "failed"}:
+            body["output"] = details or {"status": status}
+        payload = {
+            "batch": [{"id": str(uuid.uuid4()), "type": "trace-create", "timestamp": event_timestamp, "body": body}]
+        }
+        async with httpx.AsyncClient(base_url=creds.host, timeout=30) as client:
+            resp = await client.post("/api/public/ingestion", json=payload, auth=creds.auth)
+            if resp.status_code >= 400:
+                raise RuntimeError(f"Langfuse audit failed {resp.status_code}: {resp.text}")
+        return {"ok": True, "trace_id": trace_id, "status": status, "credential_source": creds.source}

evaluator/publishers/langfuse_scores.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+import httpx
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationRecord, TraceJudgeResult
+
+class LangfuseScorePublisher:
+    async def publish_trace_score(self, record: ConversationRecord, result: TraceJudgeResult):
+        if not settings.can_publish_langfuse_scores or not record.trace_id:
+            return None
+        auth = (settings.langfuse_public_key, settings.langfuse_secret_key)
+        payloads = [
+            {'traceId': record.trace_id, 'name': 'offline_judge_score', 'value': result.judgeScore, 'comment': result.rationale},
+            {'traceId': record.trace_id, 'name': 'offline_accuracy_score', 'value': result.accuracyScore, 'comment': result.rationale},
+            {'traceId': record.trace_id, 'name': 'offline_alucination_score', 'value': result.alucinationScore, 'comment': result.rationale},
+        ]
+        payloads = [payload for payload in payloads if payload['value'] is not None]
+        async with httpx.AsyncClient(base_url=settings.langfuse_host, timeout=30) as client:
+            for payload in payloads:
+                resp = await client.post('/api/public/scores', json=payload, auth=auth)
+                if resp.status_code >= 400:
+                    # Don't fail the run because score publishing is supplementary.
+                    return {'ok': False, 'status': resp.status_code, 'body': resp.text}
+        return {'ok': True}

evaluator/time_utils.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from datetime import date, datetime, time, timezone
+from zoneinfo import ZoneInfo
+
+BRT = ZoneInfo("America/Sao_Paulo")
+
+
+def as_brt(value: datetime) -> datetime:
+    """Interpret naive datetimes as America/Sao_Paulo and normalize aware values to BRT."""
+    if value.tzinfo is None:
+        return value.replace(tzinfo=BRT)
+    return value.astimezone(BRT)
+
+
+def to_utc_z(value: datetime) -> str:
+    return as_brt(value).astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def brt_day_window(reference_date: date) -> tuple[datetime, datetime]:
+    start = datetime.combine(reference_date, time(0, 0, 0), tzinfo=BRT)
+    end = datetime.combine(reference_date, time(23, 59, 59, 999999), tzinfo=BRT)
+    return start, end
+
+
+def brt_naive(value: datetime) -> datetime:
+    """Oracle TIMESTAMP storage helper: persist the wall clock in BRT without tzinfo."""
+    return as_brt(value).replace(tzinfo=None)