first commit

.env.example

@@ -0,0 +1,38 @@
+# Oracle Autonomous Database / Wallet - same pattern used by Agent Framework
+ADB_USER=admin
+ADB_PASSWORD=change-me
+ADB_DSN=oradb23ai_high
+ADB_WALLET_LOCATION=/path/to/Wallet_ORADB23ai
+ADB_WALLET_PASSWORD=change-me
+ADB_TABLE_PREFIX=AGENTFW
+
+# Langfuse collector / publisher
+ENABLE_LANGFUSE=true
+LANGFUSE_PUBLIC_KEY=pk-lf-...
+LANGFUSE_SECRET_KEY=sk-lf-...
+LANGFUSE_HOST=http://localhost:3005
+PUBLISH_LANGFUSE_SCORES=false
+
+# LLM - same style as Agent Framework env-driven config
+LLM_PROVIDER=oci_openai
+LLM_PROFILE=judge
+LLM_PROFILES_PATH=configs/llm_profiles/llm_profiles.yaml
+OCI_GENAI_BASE_URL=https://inference.generativeai.sa-saopaulo-1.oci.oraclecloud.com/openai/v1
+OCI_GENAI_MODEL_ID=meta.llama-3.3-70b-instruct
+OCI_GENAI_API_KEY=seu_token_aqui
+LLM_TEMPERATURE=0
+LLM_MAX_TOKENS=900
+LLM_TIMEOUT_SECONDS=120
+
+# Execution
+AGENTS_CONFIG_PATH=configs/judge/agents.yaml
+TRACE_PROMPT_PATH=configs/judge/trace_metrics.yaml
+SESSION_PROMPT_PATH=configs/judge/session_metrics.yaml
+OUTPUT_DIR=output
+BATCH_SIZE=50
+MAX_ATTEMPTS=3
+
+# Optional GCS export compatibility with TIM package
+ENABLE_GCS_UPLOAD=false
+JUDGE_GCS_BUCKET=
+GOOGLE_APPLICATION_CREDENTIALS=configs/GCP_ACCESS_KEY.json

.idea/.gitignore

@@ -0,0 +1,12 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Environment-dependent path to Maven home directory
+/mavenHomeManager.xml
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Zeppelin ignored files
+/ZeppelinRemoteNotebooks/

.idea/agent_framework_evaluator.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/codeStyles/Project.xml

@@ -0,0 +1,7 @@
+<component name="ProjectCodeStyleConfiguration">
+  <code_scheme name="Project" version="173">
+    <ScalaCodeStyleSettings>
+      <option name="MULTILINE_STRING_CLOSING_QUOTES_ON_NEW_LINE" value="true" />
+    </ScalaCodeStyleSettings>
+  </code_scheme>
+</component>

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
+  </state>
+</component>

.idea/misc.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_24" default="true" project-jdk-name="24" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/agent_framework_evaluator.iml" filepath="$PROJECT_DIR$/.idea/agent_framework_evaluator.iml" />
+    </modules>
+  </component>
+</project>

.oca/custom_code_review_guidelines.txt

@@ -0,0 +1,24 @@
+# Sample guideline, please follow similar structure for guideline with code samples
+# 1. Suggest using streams instead of simple loops for better readability.
+#          <example>
+#          *Comment:
+#          Category: Minor
+#          Issue: Use streams instead of a loop for better readability.
+#          Code Block:
+#
+#        ```java
+#            // Calculate squares of numbers
+#            List<Integer> squares = new ArrayList<>();
+#            for (int number : numbers) {
+#              squares.add(number * number);
+#            }
+#          ```
+#          Recommendation:
+#
+#        ```java
+#            // Calculate squares of numbers
+#            List<Integer> squares = Arrays.stream(numbers)
+#              .map(n -> n * n) // Map each number to its square
+#              .toList();
+#          ```
+#          </example>

Dockerfile

@@ -0,0 +1,5 @@
+FROM python:3.12-slim
+WORKDIR /app
+COPY . /app
+RUN pip install --no-cache-dir -e .
+CMD ["python", "-m", "evaluator.cli", "run-agents", "--source", "langfuse"]

configs/identity.yaml

@@ -0,0 +1,55 @@
+identity:
+  version: "2"
+  required:
+    - session_key
+  keys:
+    customer_key:
+      description: Cliente/assinante/consumidor canônico.
+      sources:
+        - business_context.customer_key
+        - customer_key
+        - msisdn
+        - customer_id
+        - user_id
+        - ani
+        - from
+    contract_key:
+      description: Contrato, conta, fatura, pedido ou asset principal.
+      sources:
+        - business_context.contract_key
+        - contract_key
+        - invoice_id
+        - current_invoice_number
+        - order_id
+        - pedido_id
+        - asset_id
+    interaction_key:
+      description: Chave externa da interação/call/chat vinda do canal.
+      sources:
+        - business_context.interaction_key
+        - interaction_key
+        - ura_call_id
+        - call_id
+        - message_id
+    account_key:
+      description: Conta de cobrança/conta comercial.
+      sources:
+        - business_context.account_key
+        - account_key
+        - account_id
+        - billing_account_id
+    resource_key:
+      description: Recurso/linha/produto/asset específico.
+      sources:
+        - business_context.resource_key
+        - resource_key
+        - asset_id
+        - product_id
+        - sku
+    session_key:
+      description: Sessão técnica estável já escopada por tenant e agente.
+      sources:
+        - business_context.session_key
+        - session_key
+        - conversation_key
+        - session_id

configs/judge/agents.yaml

@@ -0,0 +1,21 @@
+agents:
+  - agent_id: telecom_contas
+    enabled: true
+    days_back: 1
+    percentage: 1.0
+    langfuse_agent_aliases: [telecom_contas, billing_agent, financeiro_agent]
+    gcs_prefix: agnt_ai_contas/llm_qa/input
+
+  - agent_id: retail_orders
+    enabled: true
+    days_back: 1
+    percentage: 1.0
+    langfuse_agent_aliases: [retail_orders, orders_agent, retail_agent]
+    gcs_prefix: agnt_ai_orders/llm_qa/input
+
+  - agent_id: financeiro_agent
+    enabled: true
+    days_back: 1
+    percentage: 1.0
+    langfuse_agent_aliases: [financeiro_agent, billing_agent, telecom_contas]
+    gcs_prefix: agnt_ai_financeiro/llm_qa/input

configs/judge/session_metrics.yaml

@@ -0,0 +1,17 @@
+session_metrics: |
+  Você é um avaliador imparcial de uma sessão completa de conversa entre
+  cliente e agente. Vai receber a TRANSCRIÇÃO da sessão (alternância
+  user/agent). Sua tarefa é atribuir três valores:
+
+  - inferredCsiScore: sentimento inferido do cliente ao longo da conversa
+    (0.0 negativo, 0.5 neutro, 1.0 positivo).
+  - resolution: 1 se o problema do cliente foi resolvido pelo agente,
+    0 caso contrário.
+  - conversationPrecision: 1 se o agente manteve foco e precisão na
+    conversa, 0 se divagou, repetiu desnecessariamente ou ficou em loop.
+
+  Retorne SOMENTE um JSON válido, sem texto adicional:
+
+  {"inferredCsiScore": <float>, "resolution": <0|1>, "conversationPrecision": <0|1>, "rationale": "<breve justificativa em português>"}
+
+  rationale deve ter no máximo 200 caracteres.

configs/judge/trace_metrics.yaml

@@ -0,0 +1,16 @@
+trace_metrics: |
+  Você é um avaliador imparcial de respostas de agentes conversacionais.
+  Vai receber o HISTÓRICO da conversa (últimos turnos), a MENSAGEM DO
+  USUÁRIO e a RESPOSTA DO AGENTE. Sua tarefa é atribuir três notas
+  numéricas entre 0.0 e 1.0:
+
+  - judgeScore: qualidade global da resposta (clareza, utilidade, tom).
+  - accuracyScore: factualidade da resposta frente ao contexto fornecido.
+  - alucinationScore: grau em que a resposta contém afirmações NÃO
+    suportadas pelo contexto (alto = mais alucinação = pior).
+
+  Retorne SOMENTE um JSON válido, sem nenhum texto adicional, no formato:
+
+  {"judgeScore": <float>, "accuracyScore": <float>, "alucinationScore": <float>, "rationale": "<breve justificativa em português>"}
+
+  rationale deve ter no máximo 200 caracteres.

configs/llm_profiles/llm_profiles.yaml

@@ -0,0 +1,11 @@
+profiles:
+  judge:
+    provider: oci_openai
+    model: meta.llama-3.3-70b-instruct
+    temperature: 0
+    max_tokens: 900
+  mock:
+    provider: mock
+    model: mock-judge
+    temperature: 0
+    max_tokens: 300

evaluator/analytics/vloop.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+def _normalize(text: Any) -> str:
+    """Same deterministic spirit as Agent Framework VLOOP: lower + strip.
+
+    We also collapse whitespace because offline telemetry can contain line breaks,
+    repeated spaces, or formatting differences from Langfuse observations.
+    """
+    if text is None:
+        return ""
+    return re.sub(r"\s+", " ", str(text).lower()).strip()
+
+
+def _message_role(message: Any) -> str:
+    if isinstance(message, dict):
+        return str(message.get("role") or "").lower()
+    return str(getattr(message, "role", "") or "").lower()
+
+
+def _message_content(message: Any) -> str:
+    if isinstance(message, dict):
+        return str(message.get("content") or "")
+    return str(getattr(message, "content", "") or "")
+
+
+def user_texts_from_record(record_or_raw: Any) -> list[str]:
+    """Extract user/human texts from ConversationRecord or its JSON dict."""
+    if isinstance(record_or_raw, dict):
+        messages = record_or_raw.get("messages") or []
+        input_text = record_or_raw.get("input_text") or ""
+    else:
+        messages = getattr(record_or_raw, "messages", []) or []
+        input_text = getattr(record_or_raw, "input_text", "") or ""
+
+    out: list[str] = []
+    for message in messages:
+        role = _message_role(message)
+        if role in {"user", "human", "cliente", "customer"}:
+            text = _normalize(_message_content(message))
+            if text:
+                out.append(text)
+
+    # Ensure the canonical current user input participates even if messages were
+    # reconstructed only from observations and missed the trace-level input.
+    canonical = _normalize(input_text)
+    if canonical and canonical not in out:
+        out.append(canonical)
+    return out
+
+
+def detect_vloop(record_or_raw: Any, history_window: int = 6, min_previous_repetitions: int = 2) -> bool:
+    """Offline equivalent of Agent Framework VLOOP.
+
+    Framework logic:
+        normalized = lower(text).strip()
+        history = lower(history_texts)[-6:]
+        repeated = history.count(normalized) >= 2
+
+    Offline telemetry does not provide the exact guardrail context, so we rebuild
+    it from user messages. The current user text is the last user message. The
+    previous history is the prior messages in the same reconstructed trace/session.
+    """
+    texts = user_texts_from_record(record_or_raw)
+    if not texts:
+        return False
+
+    current = texts[-1]
+    if not current:
+        return False
+
+    history = texts[:-1][-history_window:]
+    if history.count(current) >= min_previous_repetitions:
+        return True
+
+    # Defensive fallback: if the reconstructed messages do not preserve a clear
+    # current turn, flag any user utterance repeated 3+ times in the recent window.
+    recent = texts[-(history_window + 1):]
+    return any(recent.count(t) >= (min_previous_repetitions + 1) for t in set(recent) if t)
+
+
+def vloop_flag(record_or_raw: Any) -> int:
+    return 1 if detect_vloop(record_or_raw) else 0

evaluator/api/main.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse
+from evaluator.persistence.repository import EvaluationRepository
+from fastapi import Request
+from fastapi.responses import JSONResponse
+import traceback
+
+app = FastAPI(title='Agent Framework Evaluator')
+
+@app.get('/health')
+def health(): return {'status':'ok'}
+
+@app.get('/runs')
+async def runs(limit:int=20): return await EvaluationRepository(auto_init_schema=False).alist_runs(limit)
+
+@app.get('/runs/{run_id}/progress')
+async def run_progress(run_id:str, events:int=20):
+    return await EvaluationRepository(auto_init_schema=False).aget_run_progress(run_id, events)
+
+@app.get("/runs/{run_id}/results")
+async def results(run_id: str, limit: int = 100):
+    return await EvaluationRepository(auto_init_schema=False).alist_results(run_id, limit)
+
+@app.get('/ui', response_class=HTMLResponse)
+def ui():
+    return '''<!doctype html><html><head><title>Agent Framework Evaluator</title><style>body{font-family:Arial;margin:32px}table{border-collapse:collapse;width:100%}td,th{border:1px solid #ddd;padding:8px}th{background:#eee}</style></head><body><h1>Agent Framework Evaluator</h1><p>Offline LLM-as-a-Judge with Agent Framework telemetry.</p><table id="runs"><thead><tr><th>Run</th><th>Agent</th><th>Source</th><th>Status</th><th>Total</th><th>Processed</th><th>Failed</th><th>Created</th></tr></thead><tbody></tbody></table><script>async function load(){const r=await fetch('/runs'); const data=await r.json(); document.querySelector('#runs tbody').innerHTML=data.map(x=>`<tr><td><a href="/runs/${x.run_id}/progress">${x.run_id}</a></td><td>${x.agent_id||''}</td><td>${x.source}</td><td>${x.status}</td><td>${x.total_items}</td><td>${x.processed_items}</td><td>${x.failed_items}</td><td>${x.created_at}</td></tr>`).join('')} load(); setInterval(load,5000);</script></body></html>'''
+
+@app.exception_handler(Exception)
+async def debug_exception_handler(request: Request, exc: Exception):
+    return JSONResponse(
+        status_code=500,
+        content={
+            "error": str(exc),
+            "traceback": traceback.format_exc(),
+        },
+    )

evaluator/cli.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+import asyncio
+from datetime import datetime, timedelta
+import typer
+from rich import print
+from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeElapsedColumn
+from evaluator.config.agents import load_agents
+from evaluator.engine import EvaluationEngine
+from evaluator.persistence.repository import EvaluationRepository
+from evaluator.config.settings import settings
+
+app = typer.Typer(help='Agent Framework TIM-style LLM Judge Evaluator')
+
+
+def _run_progress(coro_factory):
+    async def runner():
+        state={'run_id': None}
+        with Progress(SpinnerColumn(), TextColumn('[bold blue]{task.fields[stage]}'), BarColumn(), TextColumn('{task.completed}/{task.total}'), TextColumn('{task.percentage:>3.0f}%'), TimeElapsedColumn()) as progress:
+            task=progress.add_task('evaluation', total=1, stage='starting')
+            async def cb(event):
+                state['run_id'] = event.get('run_id') or state['run_id']
+                stage = event.get('stage','')
+                msg = event.get('message','')
+                if state['run_id']:
+                    snap = await EvaluationRepository(auto_init_schema=False).aget_run_progress(state['run_id'], event_limit=1)
+                    total=int(snap.get('total_items') or 0) or 1
+                    done=int(snap.get('done_items') or 0)
+                    progress.update(task,total=total,completed=done,stage=f'{stage}: {msg}'[:120])
+            result = await coro_factory(cb)
+            progress.update(task, completed=1, total=1, stage='finished')
+            return result
+    return asyncio.run(runner())
+
+@app.command("reset-db")
+def reset_db():
+    repo = EvaluationRepository(auto_init_schema=False)
+    repo.store.drop_schema()
+    repo.store._init_schema()
+    print({"status": "OK", "message": "Evaluator schema dropped and recreated successfully."})
+
+@app.command('init-db')
+def init_db():
+    EvaluationRepository(auto_init_schema=True)
+    print({'status':'OK','message':'schema checked/created'})
+
+@app.command('show-config')
+def show_config():
+    print({'env_path': str(settings.project_root / '.env'), 'adb_dsn': settings.ADB_DSN, 'wallet': settings.ADB_WALLET_LOCATION, 'langfuse': settings.enable_langfuse, 'publish_langfuse_scores': settings.publish_langfuse_scores, 'llm_provider': settings.llm_provider, 'llm_profile': settings.llm_profile, 'oci_genai_base_url': settings.OCI_GENAI_BASE_URL, 'oci_genai_model': settings.OCI_GENAI_MODEL, 'oci_genai_api_key_configured': bool(settings.OCI_GENAI_API_KEY), 'agents_config': settings.agents_config_path})
+
+@app.command('run')
+def run(period_start: datetime, period_end: datetime, source: str='langfuse', limit: int|None=None, show_progress: bool=True):
+    if show_progress:
+        result = _run_progress(lambda cb: EvaluationEngine(progress_callback=cb).run(period_start, period_end, source, limit))
+    else:
+        result = asyncio.run(EvaluationEngine().run(period_start, period_end, source, limit))
+    print(result)
+
+@app.command('run-agents')
+def run_agents(source: str='langfuse', agent_id: str|None=None, limit: int|None=None):
+    async def main():
+        results=[]
+        now=datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
+        for agent in load_agents():
+            if agent_id and agent.agent_id != agent_id: continue
+            start = now - timedelta(days=agent.days_back)
+            engine=EvaluationEngine()
+            results.append(await engine.run_agent(agent, start, now, source=source, limit=limit))
+        return results
+    print(asyncio.run(main()))
+
+@app.command('progress')
+def progress(run_id: str, events: int=20):
+    print(asyncio.run(EvaluationRepository(auto_init_schema=False).aget_run_progress(run_id, event_limit=events)))
+
+@app.command('runs')
+def runs(limit: int=20):
+    print(asyncio.run(EvaluationRepository(auto_init_schema=False).alist_runs(limit)))
+
+if __name__ == '__main__':
+    app()

evaluator/collectors/agent_framework.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+from datetime import datetime
+from evaluator.collectors.base import ConversationCollector
+from evaluator.core.models import ConversationRecord, ConversationMessage
+from evaluator.persistence.oracle_store import OracleStore, _json_loads
+from evaluator.config.settings import settings
+
+class AgentFrameworkCollector(ConversationCollector):
+    def __init__(self):
+        self.store = OracleStore(settings, auto_init_schema=False)
+
+    async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None):
+        return await self.store.to_thread(self._collect, period_start, period_end, agent_aliases or set(), limit or 100)
+
+    def _collect(self, period_start, period_end, aliases, limit):
+        records=[]
+        with self.store.connect() as conn:
+            cur=conn.cursor()
+            cur.execute(f"""
+                select * from (
+                    select SESSION_ID, AGENT_ID, CHANNEL, CONTEXT_JSON, METADATA_JSON, CREATED_AT
+                      from {self.store.t('AGENT_SESSION')}
+                     where CREATED_AT >= :start_at and CREATED_AT < :end_at
+                     order by CREATED_AT desc
+                ) where rownum <= :max_rows
+            """, dict(start_at=period_start, end_at=period_end, max_rows=limit))
+            sessions=cur.fetchall()
+            for session_id, agent_id, channel, ctx, meta, created_at in sessions:
+                if aliases and agent_id not in aliases: continue
+                cur.execute(f"""
+                    select ROLE, CONTENT, METADATA_JSON, CREATED_AT, MESSAGE_ID
+                      from {self.store.t('AGENT_MESSAGE')}
+                     where SESSION_ID=:session_id order by CREATED_AT
+                """, dict(session_id=session_id))
+                rows=cur.fetchall()
+                msgs=[]
+                for role, content, msg_meta, msg_created, message_id in rows:
+                    msgs.append(ConversationMessage(role=role, content=content or '', created_at=str(msg_created), metadata=_json_loads(msg_meta.read() if hasattr(msg_meta,'read') else msg_meta,{})))
+                input_text=next((m.content for m in msgs if m.role in ('user','human')), '')
+                output_text=next((m.content for m in reversed(msgs) if m.role in ('assistant','ai','agent')), '')
+                records.append(ConversationRecord(session_id=session_id, trace_id=session_id, message_id=rows[-1][4] if rows else None, agent_id=agent_id, channel=channel, input_text=input_text, output_text=output_text, messages=msgs, metadata=_json_loads(meta.read() if hasattr(meta,'read') else meta,{}), raw={}))
+        return records

evaluator/collectors/base.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from datetime import datetime
+from evaluator.core.models import ConversationRecord
+
+class ConversationCollector(ABC):
+    @abstractmethod
+    async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None) -> list[ConversationRecord]: ...

evaluator/collectors/langfuse.py

@@ -0,0 +1,355 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+import httpx
+
+from evaluator.collectors.base import ConversationCollector
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationMessage, ConversationRecord
+from evaluator.identity.resolver import IdentityResolver
+from evaluator.config.settings import settings
+
+def _iso_z(dt: datetime) -> str:
+    if dt.tzinfo is None:
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _metadata(obj: dict[str, Any] | None) -> dict[str, Any]:
+    if not isinstance(obj, dict):
+        return {}
+    meta = obj.get("metadata") or {}
+    return meta if isinstance(meta, dict) else {}
+
+
+def _first_value(*values: Any) -> Any:
+    for value in values:
+        if value not in (None, "", [], {}):
+            return value
+    return None
+
+
+def _content_to_text(value: Any) -> str:
+    """Convert Langfuse/OpenAI-style content payloads to plain text."""
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (int, float, bool)):
+        return str(value)
+    if isinstance(value, list):
+        parts: list[str] = []
+        for item in value:
+            text = _content_to_text(item)
+            if text:
+                parts.append(text)
+        return "\n".join(parts)
+    if isinstance(value, dict):
+        # OpenAI multimodal content often comes as {"type":"text","text":"..."}
+        for key in ("text", "content", "message", "value", "input", "output", "completion"):
+            if key in value:
+                text = _content_to_text(value.get(key))
+                if text:
+                    return text
+        # Chat completion response variants.
+        choices = value.get("choices")
+        if isinstance(choices, list) and choices:
+            return _content_to_text(choices[0])
+        msg = value.get("message")
+        if isinstance(msg, dict):
+            return _content_to_text(msg.get("content"))
+        return ""
+    return str(value)
+
+
+def _messages_from_value(value: Any, default_role: str) -> list[ConversationMessage]:
+    """Extract chat messages from strings/lists/dicts returned by Langfuse."""
+    if value in (None, "", [], {}):
+        return []
+
+    if isinstance(value, str):
+        text = value.strip()
+        return [ConversationMessage(role=default_role, content=text)] if text else []
+
+    if isinstance(value, dict):
+        # Common wrappers: {"messages": [...]}, {"input": ...}, {"output": ...}
+        for key in ("messages", "conversation", "chat"):
+            if isinstance(value.get(key), list):
+                return _messages_from_value(value[key], default_role)
+
+        if "role" in value and "content" in value:
+            role = str(value.get("role") or default_role)
+            content = _content_to_text(value.get("content")).strip()
+            return [ConversationMessage(role=role, content=content, metadata={"source": "langfuse"})] if content else []
+
+        text = _content_to_text(value).strip()
+        return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else []
+
+    if isinstance(value, list):
+        out: list[ConversationMessage] = []
+        for item in value:
+            out.extend(_messages_from_value(item, default_role))
+        return out
+
+    text = _content_to_text(value).strip()
+    return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else []
+
+
+def _agent_id(trace: dict[str, Any], detail: dict[str, Any] | None = None) -> str | None:
+    detail = detail or {}
+    meta = {**_metadata(trace), **_metadata(detail)}
+    return (
+        meta.get("agent_id")
+        or meta.get("agentId")
+        or meta.get("agent")
+        or detail.get("name")
+        or trace.get("name")
+    )
+
+
+def _channel(trace: dict[str, Any], detail: dict[str, Any] | None = None) -> str | None:
+    detail = detail or {}
+    meta = {**_metadata(trace), **_metadata(detail)}
+    return meta.get("channel") or meta.get("channel_id") or meta.get("channelId")
+
+
+def _observation_sort_key(obs: dict[str, Any]) -> str:
+    return str(
+        obs.get("startTime")
+        or obs.get("start_time")
+        or obs.get("createdAt")
+        or obs.get("created_at")
+        or obs.get("timestamp")
+        or ""
+    )
+
+
+class LangfuseCollector(ConversationCollector):
+    """Collect traces from Langfuse and hydrate each trace with detail/observations.
+
+    The list endpoint often returns only trace metadata. If we judge that directly,
+    prompts reach the LLM as empty conversations and the judge correctly returns
+    "Conversa vazia"/"Resposta vazia". This collector therefore fetches each trace
+    detail and observations before building ConversationRecord.
+    """
+
+    def __init__(self):
+        self.identity_resolver = IdentityResolver(settings.identity_config_path)
+
+    async def collect(
+        self,
+        period_start: datetime,
+        period_end: datetime,
+        agent_aliases: set[str] | None = None,
+        limit: int | None = None,
+    ) -> list[ConversationRecord]:
+        if not settings.can_use_langfuse:
+            raise RuntimeError(
+                "Langfuse disabled or credentials missing. Set ENABLE_LANGFUSE=true and LANGFUSE_PUBLIC_KEY/SECRET_KEY."
+            )
+
+        params = {
+            "fromTimestamp": _iso_z(period_start),
+            "toTimestamp": _iso_z(period_end),
+            "limit": limit or 100,
+        }
+        auth = (settings.langfuse_public_key, settings.langfuse_secret_key)
+        aliases = {a for a in (agent_aliases or set()) if a}
+
+        async with httpx.AsyncClient(base_url=settings.langfuse_host, timeout=60) as client:
+            response = await client.get("/api/public/traces", params=params, auth=auth)
+            if response.status_code >= 400:
+                raise RuntimeError(f"Langfuse traces API failed {response.status_code}: {response.text}")
+            payload = response.json()
+            traces = payload.get("data") or payload.get("traces") or []
+
+            records: list[ConversationRecord] = []
+            for trace in traces:
+                if not isinstance(trace, dict):
+                    continue
+                trace_id = trace.get("id")
+                detail = await self._fetch_trace_detail(client, trace_id, auth) if trace_id else {}
+                observations = await self._fetch_observations(client, trace_id, auth) if trace_id else []
+
+                agent_id = _agent_id(trace, detail)
+                if aliases and agent_id and agent_id not in aliases:
+                    continue
+
+                record = self._to_record(trace, detail, observations, agent_id)
+                # Do not send empty traces to the LLM judge. Empty records produce
+                # valid but misleading scores such as "Conversa vazia".
+                if not (record.input_text or record.output_text or record.messages):
+                    continue
+                records.append(record)
+
+        return records
+
+    async def _fetch_trace_detail(
+        self,
+        client: httpx.AsyncClient,
+        trace_id: str,
+        auth: tuple[str | None, str | None],
+    ) -> dict[str, Any]:
+        # Langfuse versions differ slightly. This endpoint works in current public API;
+        # if unavailable, the collector falls back to the list payload.
+        response = await client.get(f"/api/public/traces/{trace_id}", auth=auth)
+        if response.status_code >= 400:
+            return {}
+        payload = response.json()
+        if isinstance(payload, dict):
+            return payload.get("data") if isinstance(payload.get("data"), dict) else payload
+        return {}
+
+    async def _fetch_observations(
+        self,
+        client: httpx.AsyncClient,
+        trace_id: str,
+        auth: tuple[str | None, str | None],
+    ) -> list[dict[str, Any]]:
+        # Try common Langfuse public API shapes. Ignore failures because trace detail
+        # may already contain observations in some versions.
+        candidates = [
+            ("/api/public/observations", {"traceId": trace_id, "limit": 100}),
+            ("/api/public/observations", {"trace_id": trace_id, "limit": 100}),
+        ]
+        for path, params in candidates:
+            response = await client.get(path, params=params, auth=auth)
+            if response.status_code >= 400:
+                continue
+            payload = response.json()
+            items = payload.get("data") or payload.get("observations") or [] if isinstance(payload, dict) else []
+            if isinstance(items, list):
+                return [x for x in items if isinstance(x, dict)]
+        return []
+
+    def _to_record(
+        self,
+        trace: dict[str, Any],
+        detail: dict[str, Any],
+        observations: list[dict[str, Any]],
+        agent_id: str | None,
+    ) -> ConversationRecord:
+        meta = {**_metadata(trace), **_metadata(detail)}
+        trace_id = trace.get("id") or detail.get("id")
+        session_id = (
+            detail.get("sessionId")
+            or detail.get("session_id")
+            or trace.get("sessionId")
+            or trace.get("session_id")
+            or trace_id
+        )
+
+        # Detail payload may already include observations.
+        detail_observations = detail.get("observations") or []
+        if isinstance(detail_observations, list):
+            observations = [*observations, *[x for x in detail_observations if isinstance(x, dict)]]
+        observations = sorted(observations, key=_observation_sort_key)
+
+        input_value = _first_value(
+            detail.get("input"),
+            trace.get("input"),
+            meta.get("input"),
+            meta.get("user_message"),
+            meta.get("message"),
+            meta.get("question"),
+        )
+        output_value = _first_value(
+            detail.get("output"),
+            trace.get("output"),
+            meta.get("output"),
+            meta.get("response"),
+            meta.get("answer"),
+        )
+
+        messages: list[ConversationMessage] = []
+        messages.extend(_messages_from_value(input_value, "user"))
+        messages.extend(_messages_from_value(output_value, "assistant"))
+
+        for obs in observations:
+            obs_meta = _metadata(obs)
+            obs_kind = str(obs.get("type") or obs.get("name") or "observation").lower()
+            source_meta = {"source": "langfuse_observation", "observation_id": obs.get("id"), "observation_type": obs_kind}
+            if obs_meta:
+                source_meta["metadata"] = obs_meta
+
+            # Prefer preserving explicit chat roles from observation input.
+            before = len(messages)
+            messages.extend(_messages_from_value(obs.get("input"), "user"))
+            for m in messages[before:]:
+                m.metadata.update(source_meta)
+
+            before = len(messages)
+            default_output_role = "assistant" if obs_kind in {"generation", "span", "event", "observation"} else "assistant"
+            messages.extend(_messages_from_value(obs.get("output"), default_output_role))
+            for m in messages[before:]:
+                m.metadata.update(source_meta)
+
+        messages = self._deduplicate_messages(messages)
+
+        input_text = _content_to_text(input_value).strip()
+        output_text = _content_to_text(output_value).strip()
+
+        if not input_text:
+            first_user = next((m.content for m in messages if m.role.lower() in {"user", "human"} and m.content), "")
+            input_text = first_user.strip()
+        if not output_text:
+            last_assistant = next(
+                (m.content for m in reversed(messages) if m.role.lower() in {"assistant", "agent", "ai"} and m.content),
+                "",
+            )
+            output_text = last_assistant.strip()
+
+        raw = {"trace": trace, "detail": detail, "observations": observations}
+
+        identity_payload = {
+            **(trace.get("metadata") or {}),
+            **(trace.get("input") or {}),
+            "business_context": (trace.get("input") or {}).get("business_context") or {},
+            "session_id": trace.get("sessionId") or trace.get("id"),
+            "message_id": (trace.get("input") or {}).get("message_id"),
+            "conversation_key": (trace.get("input") or {}).get("conversation_key"),
+        }
+
+        business_context = self.identity_resolver.resolve(identity_payload)
+
+        metadata = {
+            **(trace.get("metadata") or {}),
+            "business_context": business_context,
+            "ura_call_id": business_context.get("interaction_key"),
+        }
+        channel = (
+                trace.get("metadata", {}).get("channel")
+                or trace.get("input", {}).get("channel")
+                or trace.get("input", {}).get("metadata", {}).get("channel")
+                or "web"
+        )
+
+        return ConversationRecord(
+            trace_id=trace_id,
+            session_id=business_context.get("session_key") or trace_id,
+            message_id=business_context.get("interaction_key") or trace_id,
+            agent_id=agent_id,
+            channel=channel,
+            input_text=input_text,
+            output_text=output_text,
+            messages=messages,
+            metadata=metadata,
+            raw=raw,
+        )
+
+    def _deduplicate_messages(self, messages: list[ConversationMessage]) -> list[ConversationMessage]:
+        out: list[ConversationMessage] = []
+        seen: set[tuple[str, str]] = set()
+        for msg in messages:
+            content = (msg.content or "").strip()
+            if not content:
+                continue
+            key = (msg.role.lower(), content)
+            if key in seen:
+                continue
+            seen.add(key)
+            msg.content = content
+            out.append(msg)
+        return out

evaluator/collectors/mock.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+from datetime import datetime
+from evaluator.collectors.base import ConversationCollector
+from evaluator.core.models import ConversationRecord, ConversationMessage
+
+class MockCollector(ConversationCollector):
+    async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None=None, limit: int | None=None):
+        agent = next(iter(agent_aliases), 'telecom_contas') if agent_aliases else 'telecom_contas'
+        return [ConversationRecord(trace_id='mock-trace-1', session_id='mock-session-1', message_id='mock-message-1', agent_id=agent, channel='web', input_text='quero minha fatura', output_text='Sua fatura está em aberto no valor de R$ 120.', messages=[ConversationMessage(role='user', content='quero minha fatura'), ConversationMessage(role='assistant', content='Sua fatura está em aberto no valor de R$ 120.')], metadata={'mock': True})]

evaluator/config/agents.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+from dataclasses import dataclass, field
+import yaml
+from pathlib import Path
+from evaluator.config.settings import settings
+
+@dataclass
+class AgentConfig:
+    agent_id: str
+    enabled: bool = True
+    days_back: int = 1
+    percentage: float = 1.0
+    langfuse_agent_aliases: list[str] = field(default_factory=list)
+    gcs_prefix: str = ""
+
+    @property
+    def aliases(self) -> set[str]:
+        return {self.agent_id, *self.langfuse_agent_aliases}
+
+
+def load_agents(path: str | None = None) -> list[AgentConfig]:
+    p = settings.path(path or settings.agents_config_path)
+    data = yaml.safe_load(p.read_text()) or {}
+    agents = []
+    for item in data.get("agents", []):
+        cfg = AgentConfig(
+            agent_id=item["agent_id"],
+            enabled=bool(item.get("enabled", True)),
+            days_back=int(item.get("days_back", item.get("daysBack", 1))),
+            percentage=float(item.get("percentage", 1.0)),
+            langfuse_agent_aliases=list(item.get("langfuse_agent_aliases", [])),
+            gcs_prefix=str(item.get("gcs_prefix", "")),
+        )
+        if cfg.enabled:
+            agents.append(cfg)
+    return agents

evaluator/config/settings.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from dotenv import load_dotenv
+
+ROOT_DIR = Path(__file__).resolve().parents[2]
+ENV_PATH = ROOT_DIR / ".env"
+load_dotenv(ENV_PATH, override=True)
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=str(ENV_PATH), extra="ignore", case_sensitive=False)
+
+    adb_user: str = Field(default="", validation_alias="ADB_USER")
+    adb_password: str = Field(default="", validation_alias="ADB_PASSWORD")
+    adb_dsn: str = Field(default="", validation_alias="ADB_DSN")
+    adb_wallet_location: str | None = Field(default=None, validation_alias="ADB_WALLET_LOCATION")
+    adb_wallet_password: str | None = Field(default=None, validation_alias="ADB_WALLET_PASSWORD")
+    adb_table_prefix: str = Field(default="AGENTFW", validation_alias="ADB_TABLE_PREFIX")
+
+    enable_langfuse: bool = Field(default=False, validation_alias="ENABLE_LANGFUSE")
+    langfuse_public_key: str | None = Field(default=None, validation_alias="LANGFUSE_PUBLIC_KEY")
+    langfuse_secret_key: str | None = Field(default=None, validation_alias="LANGFUSE_SECRET_KEY")
+    langfuse_host: str = Field(default="http://localhost:3005", validation_alias="LANGFUSE_HOST")
+    publish_langfuse_scores: bool = Field(default=False, validation_alias="PUBLISH_LANGFUSE_SCORES")
+
+    # llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER")
+    # llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE")
+    # llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH")
+    # oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT")
+    # oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID")
+    # oci_genai_compartment_id: str | None = Field(default=None, validation_alias="OCI_GENAI_COMPARTMENT_ID")
+    # oci_genai_auth_type: str = Field(default="api_key", validation_alias="OCI_GENAI_AUTH_TYPE")
+    # oci_config_path: str | None = Field(default=None, validation_alias="OCI_CONFIG_PATH")
+    # oci_config_profile: str = Field(default="DEFAULT", validation_alias="OCI_CONFIG_PROFILE")
+    # llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE")
+    # llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS")
+
+    # LLM / OCI GenAI OpenAI-compatible, mesmo padrão do Agent Framework
+    llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER")
+    llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE")
+    llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH")
+
+    oci_genai_base_url: str | None = Field(default=None, validation_alias="OCI_GENAI_BASE_URL")
+    oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT")  # compatibilidade
+    oci_genai_model: str = Field(default="openai.gpt-4.1", validation_alias="OCI_GENAI_MODEL")
+    oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID")  # compatibilidade
+    oci_genai_api_key: str | None = Field(default=None, validation_alias="OCI_GENAI_API_KEY")
+    oci_genai_project_ocid: str | None = Field(default=None, validation_alias="OCI_GENAI_PROJECT_OCID")
+
+    llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE")
+    llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS")
+    llm_timeout_seconds: int = Field(default=120, validation_alias="LLM_TIMEOUT_SECONDS")
+
+    agents_config_path: str = Field(default="configs/judge/agents.yaml", validation_alias="AGENTS_CONFIG_PATH")
+    trace_prompt_path: str = Field(default="configs/judge/trace_metrics.yaml", validation_alias="TRACE_PROMPT_PATH")
+    session_prompt_path: str = Field(default="configs/judge/session_metrics.yaml", validation_alias="SESSION_PROMPT_PATH")
+    output_dir: str = Field(default="output", validation_alias="OUTPUT_DIR")
+    batch_size: int = Field(default=50, validation_alias="BATCH_SIZE")
+    max_attempts: int = Field(default=3, validation_alias="MAX_ATTEMPTS")
+    enable_gcs_upload: bool = Field(default=False, validation_alias="ENABLE_GCS_UPLOAD")
+    judge_gcs_bucket: str | None = Field(default=None, validation_alias="JUDGE_GCS_BUCKET")
+    google_application_credentials: str | None = Field(default=None, validation_alias="GOOGLE_APPLICATION_CREDENTIALS")
+    identity_config_path: str = "configs/identity.yaml"
+
+    @property
+    def project_root(self) -> Path:
+        return ROOT_DIR
+
+    def path(self, value: str | Path) -> Path:
+        p = Path(value)
+        return p if p.is_absolute() else ROOT_DIR / p
+
+    @property
+    def ADB_USER(self): return self.adb_user
+    @property
+    def ADB_PASSWORD(self): return self.adb_password
+    @property
+    def ADB_DSN(self): return self.adb_dsn
+    @property
+    def ADB_WALLET_LOCATION(self): return self.adb_wallet_location
+    @property
+    def ADB_WALLET_PASSWORD(self): return self.adb_wallet_password
+    @property
+    def ADB_TABLE_PREFIX(self): return (self.adb_table_prefix or "AGENTFW").upper().rstrip("_")
+
+    @property
+    def has_langfuse_credentials(self) -> bool:
+        return bool(self.langfuse_public_key and self.langfuse_secret_key)
+
+    @property
+    def can_use_langfuse(self) -> bool:
+        return bool(self.enable_langfuse and self.has_langfuse_credentials)
+
+    @property
+    def can_publish_langfuse_scores(self) -> bool:
+        return bool(self.publish_langfuse_scores and self.can_use_langfuse)
+
+    @property
+    def OCI_GENAI_BASE_URL(self) -> str | None:
+        return self.oci_genai_base_url or self.oci_genai_endpoint
+
+    @property
+    def OCI_GENAI_MODEL(self) -> str:
+        return self.oci_genai_model_id or self.oci_genai_model
+
+    @property
+    def OCI_GENAI_API_KEY(self) -> str | None:
+        return self.oci_genai_api_key
+
+    @property
+    def OCI_GENAI_PROJECT_OCID(self) -> str | None:
+        return self.oci_genai_project_ocid
+
+    @property
+    def LLM_PROVIDER(self) -> str:
+        return self.llm_provider
+
+    @property
+    def LLM_TEMPERATURE(self) -> float:
+        return self.llm_temperature
+
+    @property
+    def LLM_MAX_TOKENS(self) -> int:
+        return self.llm_max_tokens
+
+    @property
+    def LLM_TIMEOUT_SECONDS(self) -> int:
+        return self.llm_timeout_seconds
+
+    @property
+    def LLM_PROFILES_PATH(self) -> str:
+        return self.llm_profiles_path
+
+settings = Settings()
+if settings.ADB_WALLET_LOCATION:
+    os.environ["TNS_ADMIN"] = settings.ADB_WALLET_LOCATION

evaluator/core/models.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+from datetime import datetime
+from enum import Enum
+from pydantic import BaseModel, Field
+from typing import Any
+
+class RunStatus(str, Enum):
+    RUNNING = "RUNNING"
+    COMPLETED = "COMPLETED"
+    PARTIAL = "PARTIAL"
+    FAILED = "FAILED"
+
+class ItemStatus(str, Enum):
+    PENDING = "PENDING"
+    PROCESSING = "PROCESSING"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
+    SKIPPED = "SKIPPED"
+
+class ConversationMessage(BaseModel):
+    role: str
+    content: str
+    created_at: str | None = None
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+class ConversationRecord(BaseModel):
+    trace_id: str | None = None
+    session_id: str
+    message_id: str | None = None
+    agent_id: str | None = None
+    channel: str | None = None
+    input_text: str = ""
+    output_text: str = ""
+    messages: list[ConversationMessage] = Field(default_factory=list)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    raw: dict[str, Any] = Field(default_factory=dict)
+
+class TraceJudgeResult(BaseModel):
+    judgeScore: float
+    accuracyScore: float
+    alucinationScore: float
+    rationale: str = ""
+    judge_name: str = "trace_metrics"
+    judge_type: str = "trace"
+
+class SessionJudgeResult(BaseModel):
+    inferredCsiScore: float
+    resolution: int
+    conversationPrecision: int
+    rationale: str = ""
+    judge_name: str = "session_metrics"
+    judge_type: str = "session"
+
+class CombinedJudgeResult(BaseModel):
+    trace: TraceJudgeResult
+    session: SessionJudgeResult | None = None

evaluator/engine.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+import inspect, json, random
+from datetime import datetime, timedelta
+from typing import Any, Awaitable, Callable
+from evaluator.collectors.base import ConversationCollector
+from evaluator.collectors.langfuse import LangfuseCollector
+from evaluator.collectors.agent_framework import AgentFrameworkCollector
+from evaluator.collectors.mock import MockCollector
+from evaluator.config.agents import AgentConfig
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationRecord, RunStatus
+from evaluator.judges.llm_judge import TIMStyleLLMJudge
+from evaluator.output.legacy_exporter import export_legacy_txt_gz
+from evaluator.persistence.repository import EvaluationRepository
+from evaluator.publishers.langfuse_scores import LangfuseScorePublisher
+
+ProgressCallback = Callable[[dict[str, Any]], Awaitable[None] | None]
+
+class EvaluationEngine:
+    def __init__(self, repository: EvaluationRepository | None=None, progress_callback: ProgressCallback | None=None):
+        self.repository = repository or EvaluationRepository(auto_init_schema=False)
+        self.progress_callback = progress_callback
+        self.judge = TIMStyleLLMJudge()
+        self.langfuse_publisher = LangfuseScorePublisher()
+
+    # async def _emit(self, run_id: str, stage: str, message: str='', **details):
+    #     details.pop('run_id', None)
+    #     await self.repository.arecord_progress(run_id, stage, message, details)
+    #     event={'run_id': run_id, 'stage': stage, 'message': message, 'details': details}
+    async def _emit(self, progress_run_id: str, stage: str, message: str = "", **details):
+        details.pop("run_id", None)
+
+        await self.repository.arecord_progress(
+            progress_run_id,
+            stage,
+            message,
+            details,
+        )
+
+        event = {
+            "run_id": progress_run_id,
+            "stage": stage,
+            "message": message,
+            "details": details,
+        }
+
+        if self.progress_callback:
+            r = self.progress_callback(event)
+            if inspect.isawaitable(r): await r
+
+    def collector_for(self, source: str) -> ConversationCollector:
+        if source == 'langfuse': return LangfuseCollector()
+        if source == 'agent_framework': return AgentFrameworkCollector()
+        if source == 'mock': return MockCollector()
+        raise ValueError('source must be langfuse, agent_framework or mock')
+
+    async def run_agent(self, agent: AgentConfig, period_start: datetime, period_end: datetime, source: str='langfuse', limit: int | None=None) -> dict:
+        run_id = await self.repository.acreate_run(period_start, period_end, source, agent.agent_id)
+        try:
+            await self._emit(run_id, 'RUN_CREATED', f'Agent run created: {agent.agent_id}', agent_id=agent.agent_id, source=source)
+            collector = self.collector_for(source)
+            await self._emit(run_id, 'COLLECTING', 'Collecting conversations')
+            records = await collector.collect(period_start, period_end, agent_aliases=agent.aliases, limit=limit)
+            await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} records before sampling')
+            records = self._sample(records, agent.percentage)
+            await self._emit(run_id, 'SAMPLED', f'Kept {len(records)} records', percentage=agent.percentage)
+            inserted = await self.repository.ainsert_items(run_id, records)
+            await self._emit(run_id, 'ITEMS_INSERTED', f'Inserted {inserted} items')
+            summary = await self._process(run_id)
+            output_path = export_legacy_txt_gz(self.repository, run_id, agent.agent_id)
+            await self._emit(run_id, 'EXPORTED', f'Exported {output_path}', output_file=str(output_path))
+            return {**summary, 'agent_id': agent.agent_id, 'output_file': str(output_path), 'uploaded_to': None}
+        except Exception as exc:
+            await self.repository.amark_run_status(run_id, RunStatus.PARTIAL, str(exc))
+            await self._emit(run_id, 'PARTIAL', f'Run failed: {exc}', error=str(exc))
+            return {'status':'PARTIAL','run_id':run_id,'agent_id':agent.agent_id,'error':str(exc)}
+
+    async def run(self, period_start: datetime, period_end: datetime, source: str='langfuse', limit: int | None=None) -> dict:
+        run_id = await self.repository.acreate_run(period_start, period_end, source, None)
+        try:
+            collector = self.collector_for(source)
+            await self._emit(run_id, 'COLLECTING', 'Collecting conversations')
+            records = await collector.collect(period_start, period_end, limit=limit)
+            await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} records')
+            await self.repository.ainsert_items(run_id, records)
+            return await self._process(run_id)
+        except Exception as exc:
+            await self.repository.amark_run_status(run_id, RunStatus.PARTIAL, str(exc))
+            await self._emit(run_id, 'PARTIAL', f'Run failed: {exc}', error=str(exc))
+            return {'status':'PARTIAL','run_id':run_id,'error':str(exc)}
+
+    async def _process(self, run_id: str) -> dict:
+        processed_records: list[ConversationRecord] = []
+        while True:
+            items = await self.repository.afetch_next_items(run_id, settings.batch_size)
+            if not items: break
+            await self._emit(run_id, 'BATCH_STARTED', f'Processing {len(items)} items')
+            for item in items:
+                item_id=item['item_id']
+                await self.repository.amark_item_processing(item_id)
+                try:
+                    raw=item['raw_json']
+                    if hasattr(raw, 'read'): raw = raw.read()
+                    record = ConversationRecord.model_validate(json.loads(raw))
+                    result = await self.judge.judge_trace(record)
+                    await self.repository.asave_trace_result(run_id, item_id, record, result)
+                    await self.langfuse_publisher.publish_trace_score(record, result)
+                    await self.repository.amark_item_completed(run_id, item_id)
+                    processed_records.append(record)
+
+                    #await self._emit(run_id, 'ITEM_COMPLETED', f'Item completed {item_id}', trace_id=record.trace_id)
+                    loop_result = getattr(result, "loop_result", None)
+
+                    await self._emit(
+                        run_id,
+                        "ITEM_COMPLETED",
+                        f"Item completed {item_id}",
+                        trace_id=record.trace_id,
+                        session_id=record.session_id,
+                        judgeScore=result.judgeScore,
+                        accuracyScore=result.accuracyScore,
+                        alucinationScore=result.alucinationScore,
+                        rationale=result.rationale,
+                        loop=getattr(loop_result, "loop", 0) if loop_result else 0,
+                        loop_reason=getattr(loop_result, "reason", "") if loop_result else "",
+                    )
+                except Exception as exc:
+                    await self.repository.amark_item_failed(run_id, item_id, str(exc))
+                    await self._emit(run_id, 'ITEM_FAILED', f'Item failed {item_id}', error=str(exc))
+        if processed_records:
+            sessions = await self.judge.judge_sessions(processed_records)
+            for sid, result in sessions.items():
+                agent_id = next((r.agent_id for r in processed_records if r.session_id == sid), None)
+                await self.repository.asave_session_result(run_id, sid, agent_id, result)
+            await self._emit(run_id, 'SESSION_JUDGE_COMPLETED', f'Evaluated {len(sessions)} sessions')
+        await self.repository.amark_run_status(run_id, RunStatus.COMPLETED)
+        summary = await self.repository.asummarize_run(run_id)
+        await self._emit(run_id, 'COMPLETED', 'Run completed', **summary)
+        return {'status':'COMPLETED', **summary}
+
+    def _sample(self, records: list[ConversationRecord], percentage: float) -> list[ConversationRecord]:
+        if percentage >= 1: return records
+        rng = random.Random(42)
+        return [r for r in records if rng.random() <= percentage]

evaluator/identity/resolver.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _deep_get(data: dict, path: str):
+    cur = data
+    for part in path.split("."):
+        if not isinstance(cur, dict):
+            return None
+        cur = cur.get(part)
+    return cur
+
+
+class IdentityResolver:
+    def __init__(self, path: str = "configs/identity.yaml"):
+        self.path = Path(path)
+        self.config = yaml.safe_load(self.path.read_text(encoding="utf-8")) or {}
+        self.identity = self.config.get("identity", {})
+        self.keys = self.identity.get("keys", {})
+
+    def resolve(self, payload: dict[str, Any]) -> dict[str, Any]:
+        out = {}
+
+        for key, spec in self.keys.items():
+            value = None
+            for source in spec.get("sources", []):
+                value = _deep_get(payload, source)
+                if value not in (None, ""):
+                    break
+            out[key] = str(value) if value not in (None, "") else None
+
+        out["metadata"] = {
+            "identity_version": self.identity.get("version"),
+            "identity_source": str(self.path),
+        }
+
+        return out

evaluator/judges/llm_judge.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+import json
+import re
+from collections import defaultdict
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationRecord, TraceJudgeResult, SessionJudgeResult
+from evaluator.llm.client import LLMClient, create_llm_client
+from evaluator.prompts.loader import load_prompt
+
+
+def _json_from_text(text: str) -> dict:
+    try:
+        return json.loads(text)
+    except Exception:
+        m = re.search(r"\{.*\}", text, flags=re.S)
+        if not m:
+            raise
+        return json.loads(m.group(0))
+
+
+def _history(record: ConversationRecord, max_chars: int = 6000) -> str:
+    if record.messages:
+        text = "\n".join(f"{m.role}: {m.content}" for m in record.messages)
+    else:
+        text = f"user: {record.input_text}\nagent: {record.output_text}"
+    return text[-max_chars:]
+
+
+class TIMStyleLLMJudge:
+    def __init__(self, llm: LLMClient | None = None):
+        self.llm = llm or create_llm_client()
+        self.trace_prompt = load_prompt(settings.trace_prompt_path, 'trace_metrics')
+        self.session_prompt = load_prompt(settings.session_prompt_path, 'session_metrics')
+
+    async def judge_trace(self, record: ConversationRecord) -> TraceJudgeResult:
+        prompt = f"""{self.trace_prompt}
+
+HISTÓRICO:
+{_history(record)}
+
+MENSAGEM DO USUÁRIO:
+{record.input_text}
+
+RESPOSTA DO AGENTE:
+{record.output_text}
+
+METADATA:
+{json.dumps(record.metadata, ensure_ascii=False, default=str)}
+"""
+        raw = await self.llm.complete(prompt)
+        data = _json_from_text(raw)
+        data.setdefault("judge_name", "trace_metrics")
+        data.setdefault("judge_type", "trace")
+        data.setdefault("judgeScore", data.get("judge_score", 0))
+        data.setdefault("accuracyScore", data.get("accuracy_score", 0))
+        data.setdefault("alucinationScore", data.get("alucination_score", 1))
+        data.setdefault("rationale", data.get("reasoning", ""))
+        return TraceJudgeResult(**data)
+
+    async def judge_sessions(self, records: list[ConversationRecord]) -> dict[str, SessionJudgeResult]:
+        grouped: dict[str, list[ConversationRecord]] = defaultdict(list)
+        for r in records:
+            grouped[r.session_id].append(r)
+        out = {}
+        for session_id, items in grouped.items():
+            transcript = "\n".join(_history(r, 3000) for r in items)[-9000:]
+            prompt = f"""{self.session_prompt}
+
+TRANSCRIÇÃO DA SESSÃO:
+{transcript}
+"""
+            raw = await self.llm.complete(prompt)
+            data = _json_from_text(raw)
+            data.setdefault("judge_name", "session_metrics")
+            data.setdefault("judge_type", "session")
+            data.setdefault("inferredCsiScore", data.get("inferred_csi_score", 0))
+            data.setdefault("resolution", data.get("resolution", 0))
+            data.setdefault("conversationPrecision", data.get("conversation_precision", 0))
+            data.setdefault("rationale", data.get("reasoning", ""))
+            out[session_id] = SessionJudgeResult(**data)
+        return out

evaluator/llm/client.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from evaluator.config.settings import settings
+from evaluator.llm.profile_resolver import LLMProfileResolver
+
+
+class LLMClient:
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        raise NotImplementedError
+
+
+class MockLLMClient(LLMClient):
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        if "inferredCsiScore" in prompt:
+            return json.dumps({
+                "inferredCsiScore": 0.5,
+                "resolution": 1,
+                "conversationPrecision": 1,
+                "rationale": "Avaliação mock."
+            }, ensure_ascii=False)
+
+        return json.dumps({
+            "judgeScore": 0.7,
+            "accuracyScore": 0.7,
+            "alucinationScore": 0.1,
+            "rationale": "Avaliação mock."
+        }, ensure_ascii=False)
+
+
+class OCICompatibleLLMClient(LLMClient):
+    """
+    Mesmo padrão do Agent Framework:
+    - LLM_PROVIDER=oci_openai
+    - OCI_GENAI_BASE_URL
+    - OCI_GENAI_API_KEY
+    - OCI_GENAI_MODEL
+    - llm_profiles.yaml opcional
+    """
+
+    def __init__(self):
+        self.resolver = LLMProfileResolver(settings)
+
+    async def complete(self, prompt: str, profile_name: str | None = None) -> str:
+        effective = self.resolver.resolve(profile_name or settings.llm_profile)
+
+        provider = str(effective.get("provider") or settings.LLM_PROVIDER)
+        model = str(effective.get("model") or settings.OCI_GENAI_MODEL)
+        base_url = effective.get("base_url") or settings.OCI_GENAI_BASE_URL
+        api_key = effective.get("api_key") or settings.OCI_GENAI_API_KEY
+        temperature = effective.get("temperature", settings.LLM_TEMPERATURE)
+        max_tokens = effective.get("max_tokens", settings.LLM_MAX_TOKENS)
+        timeout = effective.get("timeout_seconds", settings.LLM_TIMEOUT_SECONDS)
+
+        if provider == "mock":
+            return await MockLLMClient().complete(prompt, profile_name=profile_name)
+
+        if provider not in ("oci_openai", "openai_compatible"):
+            raise ValueError(f"Unsupported LLM provider: {provider}")
+
+        if not base_url:
+            raise RuntimeError("OCI_GENAI_BASE_URL is required for oci_openai provider")
+
+        if not api_key:
+            raise RuntimeError("OCI_GENAI_API_KEY is required for oci_openai provider")
+
+        from openai import AsyncOpenAI
+
+        client = AsyncOpenAI(
+            base_url=base_url,
+            api_key=api_key,
+            timeout=timeout,
+        )
+
+        resp = await client.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "user", "content": prompt}
+            ],
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+
+        return resp.choices[0].message.content or ""
+
+
+def create_llm_client() -> LLMClient:
+    provider = (settings.LLM_PROVIDER or "mock").lower()
+
+    if provider in ("mock", "none"):
+        return MockLLMClient()
+
+    if provider in ("oci_openai", "openai_compatible", "oci"):
+        return OCICompatibleLLMClient()
+
+    raise ValueError(f"Unsupported LLM_PROVIDER={settings.LLM_PROVIDER}")

evaluator/llm/profile_resolver.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import copy
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _canonical_profile_name(value: str | None) -> str:
+    name = (value or "default").strip()
+    name = name.replace("-", "_").replace(".", "_").replace(" ", "_")
+    name = re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
+    return re.sub(r"_+", "_", name).strip("_") or "default"
+
+
+class LLMProfileResolver:
+    def __init__(self, settings: Any):
+        self.settings = settings
+        self.path = self._find_profiles_file()
+        self.enabled = self.path is not None
+        self._profiles = self._load_profiles(self.path) if self.enabled else {}
+
+    def _find_profiles_file(self) -> Path | None:
+        root = getattr(self.settings, "project_root", Path.cwd())
+        configured = Path(getattr(self.settings, "LLM_PROFILES_PATH", "") or "").expanduser()
+        candidates = []
+        if configured:
+            candidates.append(configured if configured.is_absolute() else Path(root) / configured)
+        candidates += [
+            Path(root) / "llm_profiles.yaml",
+            Path(root) / "configs/llm_profiles/llm_profiles.yaml",
+            Path(root) / "config/llm_profiles.yaml",
+            Path("llm_profiles.yaml"),
+            Path("configs/llm_profiles/llm_profiles.yaml"),
+        ]
+        for path in candidates:
+            if path and path.exists() and path.is_file():
+                return path
+        return None
+
+    def _load_profiles(self, path: Path) -> dict[str, dict[str, Any]]:
+        # Expand ${VAR} placeholders, matching the way env-driven framework config is commonly used.
+        text = os.path.expandvars(path.read_text(encoding="utf-8"))
+        data = yaml.safe_load(text) or {}
+        raw = data.get("profiles", data)
+        profiles = {}
+        for name, value in raw.items():
+            if isinstance(value, dict):
+                profiles[_canonical_profile_name(str(name))] = dict(value)
+        return profiles
+
+    def env_defaults(self) -> dict[str, Any]:
+        return {
+            "provider": getattr(self.settings, "LLM_PROVIDER", "mock"),
+            "model": getattr(self.settings, "OCI_GENAI_MODEL", "mock-llm"),
+            "temperature": getattr(self.settings, "LLM_TEMPERATURE", 0.0),
+            "max_tokens": getattr(self.settings, "LLM_MAX_TOKENS", 900),
+            "timeout_seconds": getattr(self.settings, "LLM_TIMEOUT_SECONDS", 120),
+            "base_url": getattr(self.settings, "OCI_GENAI_BASE_URL", None),
+            "api_key": getattr(self.settings, "OCI_GENAI_API_KEY", None),
+            "project_ocid": getattr(self.settings, "OCI_GENAI_PROJECT_OCID", None),
+        }
+
+    def resolve(self, profile_name: str | None = None, **overrides) -> dict[str, Any]:
+        profile_key = _canonical_profile_name(profile_name)
+        effective = self.env_defaults()
+
+        if self.enabled:
+            effective.update(copy.deepcopy(self._profiles.get("default") or {}))
+            effective.update(copy.deepcopy(self._profiles.get(profile_key) or {}))
+
+        for key, value in overrides.items():
+            if value is not None:
+                effective[key] = value
+
+        effective["profile_name"] = profile_key
+        return effective

evaluator/output/legacy_exporter.py

@@ -0,0 +1,278 @@
+from __future__ import annotations
+
+import gzip
+from pathlib import Path
+from datetime import datetime
+from typing import Any
+import json
+
+from evaluator.config.settings import settings
+from evaluator.persistence.repository import EvaluationRepository
+from evaluator.analytics.vloop import vloop_flag
+
+HEADER = [
+    "judgeScore", "accuracyScore", "alucinationScore", "promptLength", "loop",
+    "inferredCsiScore", "resolution", "conversationPrecision",
+    "uraCallId", "channelId", "sessionId", "messageId"
+]
+
+
+def _q(v) -> str:
+    return '"' + str("" if v is None else v).replace('"', '""') + '"'
+
+
+def export_legacy_txt_gz(repo: EvaluationRepository, run_id: str, agent_id: str) -> Path:
+    output_dir = settings.path(settings.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    path = output_dir / f"AGENTE_{agent_id}_LLM_JUDGE_{datetime.now().strftime('%Y%m%d')}.TXT.GZ"
+
+    with repo.store.connect() as conn:
+        cur = conn.cursor()
+
+        cur.execute(f"""
+            select SESSION_ID, INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION
+              from {repo.store.t('EVALUATION_RESULT')}
+             where RUN_ID = :run_id
+               and JUDGE_TYPE = 'SESSION'
+        """, {"run_id": run_id})
+
+        session_metrics = {
+            sid: {
+                "inferredCsiScore": csi,
+                "resolution": res,
+                "conversationPrecision": prec,
+            }
+            for sid, csi, res, prec in cur.fetchall()
+        }
+
+        cur.execute(f"""
+            select r.TRACE_ID, r.SESSION_ID, r.JUDGE_SCORE, r.ACCURACY_SCORE,
+                   r.ALUCINATION_SCORE, r.RATIONALE, i.CHANNEL, i.MESSAGE_ID, i.RAW_JSON
+              from {repo.store.t('EVALUATION_RESULT')} r
+              left join {repo.store.t('EVALUATION_ITEM')} i on i.ITEM_ID = r.ITEM_ID
+             where r.RUN_ID = :run_id
+               and r.JUDGE_TYPE = 'TRACE'
+             order by r.CREATED_AT
+        """, {"run_id": run_id})
+
+        rows = cur.fetchall()
+
+    with gzip.open(path, "wt", encoding="utf-8") as f:
+        for trace_id, session_id, judge, accuracy, alucination, rationale, channel, message_id, raw_json in rows:
+            session = session_metrics.get(session_id, {})
+
+            raw: dict[str, Any] = {}
+            ura_call_id = ""
+            channel_id = channel or ""
+            prompt_length = 0
+            loop = 0
+
+            try:
+                from evaluator.persistence.oracle_store import _json_loads
+
+                # raw = _json_loads(
+                #     raw_json.read() if hasattr(raw_json, "read") else raw_json,
+                #     {},
+                # )
+                raw = normalize_raw(raw_json)
+
+                metadata = raw.get("metadata") or {}
+
+                channel_id = (
+                        metadata.get("channel_id")
+                        or metadata.get("channelId")
+                        or metadata.get("channel")
+                        or channel_id
+                )
+
+                ura_call_id = extract_ura_call_id(raw, metadata, message_id)
+                prompt_length = extract_prompt_length(raw)
+                loop = vloop_flag(raw)
+
+                # print(
+                #     "[DEBUG promptLength]",
+                #     "trace_id=", trace_id,
+                #     "type(raw)=", type(raw),
+                #     "keys=", list(raw.keys())[:20] if isinstance(raw, dict) else None,
+                #     "prompt_length=", prompt_length,
+                #     "input_text_len=", len(str(raw.get("input_text") or "")) if isinstance(raw, dict) else None,
+                #     "messages=", len(raw.get("messages") or []) if isinstance(raw, dict) else None,
+                # )
+
+            except Exception as exc:
+                print(f"[legacy_exporter] metadata extraction failed trace_id={trace_id}: {exc}")
+
+            vals = [
+                judge,
+                accuracy,
+                alucination,
+                prompt_length,
+                loop,
+                session.get("inferredCsiScore"),
+                session.get("resolution"),
+                session.get("conversationPrecision"),
+                ura_call_id,
+                channel_id,
+                session_id,
+                message_id or trace_id,
+                ]
+
+            f.write("|;".join(_q(v) for v in vals) + "\n")
+
+        f.write("|;".join([_q("TOTAL"), _q(len(rows))]) + "\n")
+
+    return path
+
+def extract_ura_call_id(raw: dict, metadata: dict | None = None, message_id: str | None = None) -> str:
+    metadata = metadata or {}
+
+    business_context = (
+            metadata.get("business_context")
+            or metadata.get("businessContext")
+            or raw.get("business_context")
+            or raw.get("businessContext")
+            or raw.get("metadata", {}).get("business_context")
+            or raw.get("metadata", {}).get("businessContext")
+            or {}
+    )
+
+    if not isinstance(business_context, dict):
+        business_context = {}
+
+    trace = raw.get("raw", {}).get("trace", {}) or raw.get("trace", {}) or {}
+    detail = raw.get("raw", {}).get("detail", {}) or raw.get("detail", {}) or {}
+
+    trace_input = trace.get("input") or {}
+    detail_input = detail.get("input") or {}
+
+    trace_metadata = trace.get("metadata") or {}
+    detail_metadata = detail.get("metadata") or {}
+
+    trace_bc = trace_input.get("business_context") or {}
+    detail_bc = detail_input.get("business_context") or {}
+
+    return str(
+        business_context.get("interaction_key")
+        or business_context.get("ura_call_id")
+        or metadata.get("ura_call_id")
+        or metadata.get("uraCallId")
+        or metadata.get("interaction_key")
+        or trace_metadata.get("ura_call_id")
+        or detail_metadata.get("ura_call_id")
+        or trace_bc.get("interaction_key")
+        or detail_bc.get("interaction_key")
+        or message_id
+        or ""
+    )
+
+def normalize_raw(raw):
+    if hasattr(raw, "read"):
+        raw = raw.read()
+
+    if isinstance(raw, bytes):
+        raw = raw.decode("utf-8")
+
+    if isinstance(raw, str):
+        raw = raw.strip()
+        if not raw:
+            return {}
+        raw = json.loads(raw)
+
+    # caso esteja duplamente serializado
+    if isinstance(raw, str):
+        raw = json.loads(raw)
+
+    return raw if isinstance(raw, dict) else {}
+
+def extract_prompt_length(raw: dict) -> int:
+    # 1. tokens reais do Langfuse/framework
+    tokens = find_prompt_tokens(raw)
+    if tokens > 0:
+        return tokens
+
+    # 2. input_size dos spans
+    input_size = find_input_size(raw)
+    if input_size > 0:
+        return input_size
+
+    # 3. fallback garantido pelo ConversationRecord
+    return (
+            len(str(raw.get("input_text") or ""))
+            + len(str(raw.get("output_text") or ""))
+            + sum(
+        len(str(m.get("content") or ""))
+        for m in raw.get("messages", [])
+        if isinstance(m, dict)
+    )
+    )
+
+def _walk(obj):
+    if isinstance(obj, dict):
+        yield obj
+        for value in obj.values():
+            yield from _walk(value)
+    elif isinstance(obj, list):
+        for item in obj:
+            yield from _walk(item)
+
+
+def _to_positive_int(value) -> int:
+    try:
+        n = int(value)
+        return n if n > 0 else 0
+    except Exception:
+        return 0
+
+
+def find_prompt_tokens(raw: dict) -> int:
+    candidates = []
+
+    for obj in _walk(raw):
+        for key in (
+                "prompt_tokens",
+                "promptTokens",
+                "input_tokens",
+                "inputTokens",
+        ):
+            n = _to_positive_int(obj.get(key))
+            if n:
+                candidates.append(n)
+
+        usage = obj.get("usage")
+        if isinstance(usage, dict):
+            for key in ("input", "prompt_tokens", "promptTokens", "input_tokens", "inputTokens"):
+                n = _to_positive_int(usage.get(key))
+                if n:
+                    candidates.append(n)
+
+        usage_details = obj.get("usageDetails") or obj.get("usage_details")
+        if isinstance(usage_details, dict):
+            for key in ("input", "prompt_tokens", "promptTokens", "input_tokens", "inputTokens"):
+                n = _to_positive_int(usage_details.get(key))
+                if n:
+                    candidates.append(n)
+
+    return max(candidates) if candidates else 0
+
+
+def find_input_size(raw: dict) -> int:
+    candidates = []
+
+    for obj in _walk(raw):
+        for key in ("input_size", "inputSize"):
+            n = _to_positive_int(obj.get(key))
+            if n:
+                candidates.append(n)
+
+    return max(candidates) if candidates else 0
+
+def calculate_text_length(raw: dict) -> int:
+    return (
+            len(str(raw.get("input_text") or ""))
+            + len(str(raw.get("output_text") or ""))
+            + sum(
+        len(str(m.get("content") or ""))
+        for m in raw.get("messages", [])
+        if isinstance(m, dict)
+    )
+    )

evaluator/persistence/oracle_store.py

@@ -0,0 +1,282 @@
+from __future__ import annotations
+
+import asyncio
+import json
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+
+def _json_dumps(value: Any) -> str:
+    return json.dumps(value or {}, ensure_ascii=False, default=str)
+
+
+def _json_loads(value: str | bytes | None, default: Any):
+    if value is None:
+        return default
+    if isinstance(value, bytes):
+        value = value.decode("utf-8")
+    try:
+        return json.loads(value)
+    except Exception:
+        return default
+
+
+@dataclass
+class OracleSettings:
+    user: str
+    password: str
+    dsn: str
+    wallet_location: str | None = None
+    wallet_password: str | None = None
+    table_prefix: str = "AGENTFW"
+
+
+class OracleStore:
+    """Oracle Autonomous Database store following the Agent Framework pattern.
+
+    Uses direct oracledb.connect() with wallet arguments. Synchronous DB calls are
+    exposed through asyncio.to_thread(), matching the main framework style.
+    """
+
+    def __init__(self, settings, auto_init_schema: bool = False):
+        self.settings = settings
+        self.cfg = OracleSettings(
+            user=settings.ADB_USER or "",
+            password=settings.ADB_PASSWORD or "",
+            dsn=settings.ADB_DSN or "",
+            wallet_location=getattr(settings, "ADB_WALLET_LOCATION", None),
+            wallet_password=getattr(settings, "ADB_WALLET_PASSWORD", None),
+            table_prefix=(getattr(settings, "ADB_TABLE_PREFIX", "AGENTFW") or "AGENTFW").upper().rstrip("_"),
+        )
+        if not self.cfg.user or not self.cfg.password or not self.cfg.dsn:
+            raise RuntimeError("ADB_USER, ADB_PASSWORD and ADB_DSN are required")
+        if auto_init_schema:
+            self._init_schema()
+
+    @staticmethod
+    def now() -> datetime:
+        return datetime.now(timezone.utc)
+
+    def t(self, name: str) -> str:
+        return f"{self.cfg.table_prefix}_{name}".upper()
+
+    @contextmanager
+    def connect(self):
+        import oracledb
+        oracledb.defaults.fetch_lobs = False
+        kwargs = {}
+        if self.cfg.wallet_location:
+            kwargs["config_dir"] = self.cfg.wallet_location
+            kwargs["wallet_location"] = self.cfg.wallet_location
+        if self.cfg.wallet_password:
+            kwargs["wallet_password"] = self.cfg.wallet_password
+        conn = oracledb.connect(user=self.cfg.user, password=self.cfg.password, dsn=self.cfg.dsn, **kwargs)
+        try:
+            yield conn
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+        finally:
+            conn.close()
+
+    async def to_thread(self, func, *args, **kwargs):
+        return await asyncio.to_thread(func, *args, **kwargs)
+
+    def _exec_ddl_ignore_exists(self, cur, ddl: str):
+        try:
+            cur.execute(ddl)
+        except Exception as exc:
+            msg = str(exc)
+            if "ORA-00955" in msg or "ORA-01408" in msg or "ORA-02275" in msg:
+                return
+            raise
+
+    def _column_exists(self, cur, table_name: str, column_name: str) -> bool:
+        cur.execute("""
+            select count(*)
+              from user_tab_columns
+             where table_name = :table_name
+               and column_name = :column_name
+        """, {"table_name": self.t(table_name), "column_name": column_name.upper()})
+        return int(cur.fetchone()[0] or 0) > 0
+
+    def _ensure_column(self, cur, table_name: str, column_name: str, ddl_type: str):
+        if not self._column_exists(cur, table_name, column_name):
+            cur.execute(f"alter table {self.t(table_name)} add {column_name.upper()} {ddl_type}")
+
+    def drop_schema(self):
+        tables = [
+            "EVALUATION_RESULT",
+            "EVALUATION_FINDING",
+            "EVALUATION_METRIC",
+            "EVALUATION_ITEM",
+            "EVALUATION_PROGRESS_EVENT",
+            "EVALUATION_RUN",
+        ]
+        with self.connect() as conn:
+            cur = conn.cursor()
+            for table in tables:
+                try:
+                    cur.execute(f"drop table {self.t(table)} cascade constraints purge")
+                except Exception as exc:
+                    if "ORA-00942" in str(exc):
+                        continue
+                    raise
+
+    def _init_schema(self):
+        with self.connect() as conn:
+            cur = conn.cursor()
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_RUN')} (
+                    RUN_ID varchar2(64) primary key,
+                    AGENT_ID varchar2(128),
+                    SOURCE varchar2(64),
+                    PERIOD_START timestamp with time zone,
+                    PERIOD_END timestamp with time zone,
+                    STATUS varchar2(32) not null,
+                    TOTAL_ITEMS number default 0 not null,
+                    PROCESSED_ITEMS number default 0 not null,
+                    FAILED_ITEMS number default 0 not null,
+                    RETRY_COUNT number default 0 not null,
+                    ERROR_MESSAGE clob,
+                    LAST_HEARTBEAT_AT timestamp with time zone,
+                    CREATED_AT timestamp with time zone not null,
+                    UPDATED_AT timestamp with time zone not null
+                )
+            """)
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_PROGRESS_EVENT')} (
+                    ID number generated always as identity primary key,
+                    RUN_ID varchar2(64) not null,
+                    STAGE varchar2(128) not null,
+                    MESSAGE varchar2(1000),
+                    DETAILS_JSON clob check (DETAILS_JSON is json),
+                    CREATED_AT timestamp with time zone not null,
+                    constraint {self.t('FK_EVAL_PROGRESS_RUN')}
+                        foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID)
+                )
+            """)
+            self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_PROGRESS_RUN')} on {self.t('EVALUATION_PROGRESS_EVENT')}(RUN_ID, CREATED_AT)")
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_ITEM')} (
+                    ITEM_ID varchar2(64) primary key,
+                    RUN_ID varchar2(64) not null,
+                    TRACE_ID varchar2(256),
+                    SESSION_ID varchar2(256),
+                    MESSAGE_ID varchar2(256),
+                    AGENT_ID varchar2(128),
+                    CHANNEL varchar2(64),
+                    STATUS varchar2(32) not null,
+                    ATTEMPT_COUNT number default 0 not null,
+                    ERROR_MESSAGE clob,
+                    RAW_JSON clob check (RAW_JSON is json),
+                    CREATED_AT timestamp with time zone not null,
+                    UPDATED_AT timestamp with time zone not null,
+                    constraint {self.t('FK_EVAL_ITEM_RUN')}
+                        foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID)
+                )
+            """)
+            self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_ITEM_RUN')} on {self.t('EVALUATION_ITEM')}(RUN_ID, STATUS, CREATED_AT)")
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_RESULT')} (
+                    RESULT_ID varchar2(64) primary key,
+                    RUN_ID varchar2(64) not null,
+                    ITEM_ID varchar2(64),
+                    TRACE_ID varchar2(256),
+                    SESSION_ID varchar2(256),
+                    AGENT_ID varchar2(128),
+                    JUDGE_NAME varchar2(128) not null,
+                    JUDGE_TYPE varchar2(32),
+                    SCORE number,
+                    JUDGE_SCORE number,
+                    ACCURACY_SCORE number,
+                    ALUCINATION_SCORE number,
+                    HALLUCINATION_SCORE number,
+                    INFERRED_CSI_SCORE number,
+                    RESOLUTION number,
+                    CONVERSATION_PRECISION number,
+                    TOOL_USAGE_SCORE number,
+                    ROUTING_SCORE number,
+                    RATIONALE clob,
+                    REASONING clob,
+                    RESULT_JSON clob check (RESULT_JSON is json),
+                    CREATED_AT timestamp with time zone not null,
+                    constraint {self.t('FK_EVAL_RESULT_RUN')}
+                        foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID),
+                    constraint {self.t('FK_EVAL_RESULT_ITEM')}
+                        foreign key (ITEM_ID) references {self.t('EVALUATION_ITEM')}(ITEM_ID)
+                )
+            """)
+            self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_RESULT_RUN')} on {self.t('EVALUATION_RESULT')}(RUN_ID, ITEM_ID)")
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_METRIC')} (
+                    METRIC_ID varchar2(64) primary key,
+                    RUN_ID varchar2(64) not null,
+                    ITEM_ID varchar2(64),
+                    METRIC_NAME varchar2(128) not null,
+                    METRIC_VALUE number,
+                    DIMENSIONS_JSON clob check (DIMENSIONS_JSON is json),
+                    CREATED_AT timestamp with time zone not null,
+                    constraint {self.t('FK_EVAL_METRIC_RUN')}
+                        foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID)
+                )
+            """)
+
+            self._exec_ddl_ignore_exists(cur, f"""
+                create table {self.t('EVALUATION_FINDING')} (
+                    FINDING_ID varchar2(64) primary key,
+                    RUN_ID varchar2(64) not null,
+                    ITEM_ID varchar2(64),
+                    SEVERITY varchar2(32),
+                    CATEGORY varchar2(128),
+                    TITLE varchar2(512),
+                    DESCRIPTION clob,
+                    EVIDENCE_JSON clob check (EVIDENCE_JSON is json),
+                    CREATED_AT timestamp with time zone not null,
+                    constraint {self.t('FK_EVAL_FINDING_RUN')}
+                        foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID)
+                )
+            """)
+
+            # Non-destructive compatibility for older generated schemas.
+            for col, typ in [
+                ("RETRY_COUNT", "number default 0"),
+                ("ERROR_MESSAGE", "clob"),
+                ("LAST_HEARTBEAT_AT", "timestamp with time zone"),
+                ("UPDATED_AT", "timestamp with time zone"),
+            ]:
+                self._ensure_column(cur, "EVALUATION_RUN", col, typ)
+            for col, typ in [
+                ("ID", "number generated always as identity"),
+            ]:
+                # Identity column cannot always be added cleanly; ignore if table is old without ID.
+                try:
+                    self._ensure_column(cur, "EVALUATION_PROGRESS_EVENT", col, typ)
+                except Exception:
+                    pass
+            for col, typ in [
+                ("JUDGE_NAME", "varchar2(128) default 'unknown_judge' not null"),
+                ("JUDGE_TYPE", "varchar2(32)"),
+                ("SCORE", "number"),
+                ("JUDGE_SCORE", "number"),
+                ("ACCURACY_SCORE", "number"),
+                ("ALUCINATION_SCORE", "number"),
+                ("HALLUCINATION_SCORE", "number"),
+                ("INFERRED_CSI_SCORE", "number"),
+                ("RESOLUTION", "number"),
+                ("CONVERSATION_PRECISION", "number"),
+                ("TOOL_USAGE_SCORE", "number"),
+                ("ROUTING_SCORE", "number"),
+                ("RATIONALE", "clob"),
+                ("REASONING", "clob"),
+                ("RESULT_JSON", "clob"),
+            ]:
+                self._ensure_column(cur, "EVALUATION_RESULT", col, typ)

evaluator/persistence/repository.py

@@ -0,0 +1,410 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationRecord, ItemStatus, RunStatus, TraceJudgeResult, SessionJudgeResult
+from evaluator.persistence.oracle_store import OracleStore, _json_dumps, _json_loads
+
+
+class EvaluationRepository:
+    def __init__(self, auto_init_schema: bool = False):
+        self.store = OracleStore(settings, auto_init_schema=auto_init_schema)
+
+    def create_run(self, period_start: datetime, period_end: datetime, source: str, agent_id: str | None = None) -> str:
+        run_id = str(uuid.uuid4())
+        now = self.store.now()
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                insert into {self.store.t('EVALUATION_RUN')}
+                (RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS, TOTAL_ITEMS,
+                 PROCESSED_ITEMS, FAILED_ITEMS, RETRY_COUNT, LAST_HEARTBEAT_AT, CREATED_AT, UPDATED_AT)
+                values (:run_id, :agent_id, :period_start, :period_end, :source, :status,
+                        0, 0, 0, 0, :heartbeat_at, :created_at, :updated_at)
+            """, {
+                "run_id": run_id,
+                "agent_id": agent_id,
+                "period_start": period_start,
+                "period_end": period_end,
+                "source": source,
+                "status": RunStatus.RUNNING.value,
+                "heartbeat_at": now,
+                "created_at": now,
+                "updated_at": now,
+            })
+        return run_id
+
+    async def acreate_run(self, *args, **kwargs):
+        return await self.store.to_thread(self.create_run, *args, **kwargs)
+
+    def record_progress(self, run_id: str, stage: str, message: str = "", details: dict | None = None):
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                insert into {self.store.t('EVALUATION_PROGRESS_EVENT')}
+                (RUN_ID, STAGE, MESSAGE, DETAILS_JSON, CREATED_AT)
+                values (:run_id, :stage, :message, :details_json, :created_at)
+            """, {
+                "run_id": run_id,
+                "stage": stage,
+                "message": (message or "")[:1000],
+                "details_json": _json_dumps(details or {}),
+                "created_at": self.store.now(),
+            })
+
+    async def arecord_progress(self, *args, **kwargs):
+        return await self.store.to_thread(self.record_progress, *args, **kwargs)
+
+    def insert_items(self, run_id: str, records: list[ConversationRecord]) -> int:
+        inserted = 0
+        now = self.store.now()
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            for record in records:
+                try:
+                    cur.execute(f"""
+                        insert into {self.store.t('EVALUATION_ITEM')}
+                        (ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL,
+                         STATUS, ATTEMPT_COUNT, RAW_JSON, CREATED_AT, UPDATED_AT)
+                        values (:item_id, :run_id, :trace_id, :session_id, :message_id, :agent_id,
+                                :channel, :status, 0, :raw_json, :created_at, :updated_at)
+                    """, {
+                        "item_id": str(uuid.uuid4()),
+                        "run_id": run_id,
+                        "trace_id": record.trace_id,
+                        "session_id": record.session_id,
+                        "message_id": record.message_id,
+                        "agent_id": record.agent_id,
+                        "channel": record.channel,
+                        "status": ItemStatus.PENDING.value,
+                        "raw_json": record.model_dump_json(),
+                        "created_at": now,
+                        "updated_at": now,
+                    })
+                    inserted += 1
+                except Exception as exc:
+                    if "ORA-00001" not in str(exc):
+                        raise
+            cur.execute(f"""
+                update {self.store.t('EVALUATION_RUN')}
+                   set TOTAL_ITEMS = (
+                       select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id
+                   ),
+                   UPDATED_AT = :updated_at
+                 where RUN_ID = :run_id
+            """, {"run_id": run_id, "updated_at": self.store.now()})
+        return inserted
+
+    async def ainsert_items(self, *args, **kwargs):
+        return await self.store.to_thread(self.insert_items, *args, **kwargs)
+
+    def fetch_next_items(self, run_id: str, batch_size: int) -> list[dict]:
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                select * from (
+                    select ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL,
+                           STATUS, ATTEMPT_COUNT, RAW_JSON
+                      from {self.store.t('EVALUATION_ITEM')}
+                     where RUN_ID = :run_id
+                       and STATUS in (:pending, :failed)
+                       and ATTEMPT_COUNT < :max_attempts
+                     order by CREATED_AT
+                ) where rownum <= :batch_size
+            """, {
+                "run_id": run_id,
+                "pending": ItemStatus.PENDING.value,
+                "failed": ItemStatus.FAILED.value,
+                "max_attempts": settings.max_attempts,
+                "batch_size": batch_size,
+            })
+            cols = [d[0].lower() for d in cur.description]
+            return [dict(zip(cols, row)) for row in cur.fetchall()]
+
+    async def afetch_next_items(self, *args, **kwargs):
+        return await self.store.to_thread(self.fetch_next_items, *args, **kwargs)
+
+    def mark_item_processing(self, item_id: str):
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                update {self.store.t('EVALUATION_ITEM')}
+                   set STATUS = :status,
+                       ATTEMPT_COUNT = ATTEMPT_COUNT + 1,
+                       UPDATED_AT = :updated_at
+                 where ITEM_ID = :item_id
+            """, {
+                "status": ItemStatus.PROCESSING.value,
+                "updated_at": self.store.now(),
+                "item_id": item_id,
+            })
+
+    async def amark_item_processing(self, *args, **kwargs):
+        return await self.store.to_thread(self.mark_item_processing, *args, **kwargs)
+
+    def mark_item_completed(self, run_id: str, item_id: str):
+        now = self.store.now()
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                update {self.store.t('EVALUATION_ITEM')}
+                   set STATUS = :status,
+                       UPDATED_AT = :updated_at
+                 where ITEM_ID = :item_id
+            """, {
+                "status": ItemStatus.COMPLETED.value,
+                "updated_at": now,
+                "item_id": item_id,
+            })
+            self._refresh_run_counters(cur, run_id, now)
+
+    async def amark_item_completed(self, *args, **kwargs):
+        return await self.store.to_thread(self.mark_item_completed, *args, **kwargs)
+
+    def mark_item_failed(self, run_id: str, item_id: str, error: str):
+        now = self.store.now()
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                update {self.store.t('EVALUATION_ITEM')}
+                   set STATUS = :status,
+                       ERROR_MESSAGE = :error,
+                       UPDATED_AT = :updated_at
+                 where ITEM_ID = :item_id
+            """, {
+                "status": ItemStatus.FAILED.value,
+                "error": (error or "")[:4000],
+                "updated_at": now,
+                "item_id": item_id,
+            })
+            self._refresh_run_counters(cur, run_id, now)
+
+    async def amark_item_failed(self, *args, **kwargs):
+        return await self.store.to_thread(self.mark_item_failed, *args, **kwargs)
+
+    def _refresh_run_counters(self, cur, run_id: str, updated_at):
+        cur.execute(f"""
+            update {self.store.t('EVALUATION_RUN')}
+               set PROCESSED_ITEMS = (
+                   select count(*) from {self.store.t('EVALUATION_ITEM')}
+                    where RUN_ID = :run_id and STATUS = :completed
+               ),
+               FAILED_ITEMS = (
+                   select count(*) from {self.store.t('EVALUATION_ITEM')}
+                    where RUN_ID = :run_id and STATUS = :failed
+               ),
+               UPDATED_AT = :updated_at
+             where RUN_ID = :run_id
+        """, {
+            "run_id": run_id,
+            "completed": ItemStatus.COMPLETED.value,
+            "failed": ItemStatus.FAILED.value,
+            "updated_at": updated_at,
+        })
+
+    def save_trace_result(self, run_id: str, item_id: str, record: ConversationRecord, result: TraceJudgeResult):
+        judge_name = getattr(result, "judge_name", None) or "trace_metrics"
+        judge_type = (getattr(result, "judge_type", None) or "TRACE").upper()
+        score = getattr(result, "judgeScore", None)
+        accuracy = getattr(result, "accuracyScore", None)
+        alucination = getattr(result, "alucinationScore", None)
+        rationale = getattr(result, "rationale", None) or ""
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                insert into {self.store.t('EVALUATION_RESULT')}
+                (RESULT_ID, RUN_ID, ITEM_ID, TRACE_ID, SESSION_ID, AGENT_ID, JUDGE_NAME,
+                 JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE, ALUCINATION_SCORE,
+                 RATIONALE, RESULT_JSON, CREATED_AT)
+                values (:result_id, :run_id, :item_id, :trace_id, :session_id, :agent_id,
+                        :judge_name, :judge_type, :score, :judge_score, :accuracy_score,
+                        :alucination_score, :rationale, :result_json, :created_at)
+            """, {
+                "result_id": str(uuid.uuid4()),
+                "run_id": run_id,
+                "item_id": item_id,
+                "trace_id": record.trace_id,
+                "session_id": record.session_id,
+                "agent_id": record.agent_id,
+                "judge_name": judge_name,
+                "judge_type": judge_type,
+                "score": score,
+                "judge_score": score,
+                "accuracy_score": accuracy,
+                "alucination_score": alucination,
+                "rationale": rationale,
+                "result_json": result.model_dump_json(),
+                "created_at": self.store.now(),
+            })
+
+    async def asave_trace_result(self, *args, **kwargs):
+        return await self.store.to_thread(self.save_trace_result, *args, **kwargs)
+
+    def save_session_result(self, run_id: str, session_id: str, agent_id: str | None, result: SessionJudgeResult):
+        judge_name = getattr(result, "judge_name", None) or "session_metrics"
+        judge_type = (getattr(result, "judge_type", None) or "SESSION").upper()
+        rationale = getattr(result, "rationale", None) or ""
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                insert into {self.store.t('EVALUATION_RESULT')}
+                (RESULT_ID, RUN_ID, SESSION_ID, AGENT_ID, JUDGE_NAME, JUDGE_TYPE,
+                 INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION, RATIONALE,
+                 RESULT_JSON, CREATED_AT)
+                values (:result_id, :run_id, :session_id, :agent_id, :judge_name, :judge_type,
+                        :csi, :resolution, :precision, :rationale, :result_json, :created_at)
+            """, {
+                "result_id": str(uuid.uuid4()),
+                "run_id": run_id,
+                "session_id": session_id,
+                "agent_id": agent_id,
+                "judge_name": judge_name,
+                "judge_type": judge_type,
+                "csi": getattr(result, "inferredCsiScore", None),
+                "resolution": getattr(result, "resolution", None),
+                "precision": getattr(result, "conversationPrecision", None),
+                "rationale": rationale,
+                "result_json": result.model_dump_json(),
+                "created_at": self.store.now(),
+            })
+
+    async def asave_session_result(self, *args, **kwargs):
+        return await self.store.to_thread(self.save_session_result, *args, **kwargs)
+
+    def mark_run_status(self, run_id: str, status: RunStatus, error: str | None = None):
+        with self.store.connect() as conn:
+            conn.cursor().execute(f"""
+                update {self.store.t('EVALUATION_RUN')}
+                   set STATUS = :status,
+                       ERROR_MESSAGE = :error,
+                       UPDATED_AT = :updated_at
+                 where RUN_ID = :run_id
+            """, {
+                "status": status.value,
+                "error": error,
+                "updated_at": self.store.now(),
+                "run_id": run_id,
+            })
+
+    async def amark_run_status(self, *args, **kwargs):
+        return await self.store.to_thread(self.mark_run_status, *args, **kwargs)
+
+    def summarize_run(self, run_id: str) -> dict:
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                select
+                    (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id),
+                    (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'COMPLETED'),
+                    (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'FAILED'),
+                    (select count(*) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE'),
+                    (select avg(JUDGE_SCORE) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE')
+                from dual
+            """, {"run_id": run_id})
+            r = cur.fetchone()
+            return {
+                "run_id": run_id,
+                "total_items": int(r[0] or 0),
+                "completed_items": int(r[1] or 0),
+                "failed_items": int(r[2] or 0),
+                "evaluations": int(r[3] or 0),
+                "avg_score": float(r[4]) if r[4] is not None else None,
+            }
+
+    async def asummarize_run(self, *args, **kwargs):
+        return await self.store.to_thread(self.summarize_run, *args, **kwargs)
+
+    def get_run_progress(self, run_id: str, event_limit: int = 20) -> dict:
+        summary = self.summarize_run(run_id)
+        total = summary["total_items"] or 0
+        done = summary["completed_items"] + summary["failed_items"]
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                select * from (
+                    select STAGE, MESSAGE, DETAILS_JSON, CREATED_AT
+                      from {self.store.t('EVALUATION_PROGRESS_EVENT')}
+                     where RUN_ID = :run_id
+                     order by CREATED_AT desc
+                ) where rownum <= :max_rows
+            """, {"run_id": run_id, "max_rows": event_limit})
+            events = [
+                {
+                    "stage": s,
+                    "message": m,
+                    "details": _json_loads(d.read() if hasattr(d, "read") else d, {}),
+                    "created_at": str(c),
+                }
+                for s, m, d, c in cur.fetchall()
+            ]
+        return {
+            **summary,
+            "done_items": done,
+            "percent_complete": round((done / total) * 100, 2) if total else 0.0,
+            "events": events,
+        }
+
+    async def aget_run_progress(self, *args, **kwargs):
+        return await self.store.to_thread(self.get_run_progress, *args, **kwargs)
+
+    def list_runs(self, limit: int = 20):
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                select * from (
+                    select RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS,
+                           TOTAL_ITEMS, PROCESSED_ITEMS, FAILED_ITEMS, CREATED_AT, UPDATED_AT
+                      from {self.store.t('EVALUATION_RUN')}
+                     order by CREATED_AT desc
+                ) where rownum <= :max_rows
+            """, {"max_rows": limit})
+            return [
+                {
+                    "run_id": r[0],
+                    "agent_id": r[1],
+                    "period_start": str(r[2]),
+                    "period_end": str(r[3]),
+                    "source": r[4],
+                    "status": r[5],
+                    "total_items": int(r[6] or 0),
+                    "processed_items": int(r[7] or 0),
+                    "failed_items": int(r[8] or 0),
+                    "created_at": str(r[9]),
+                    "updated_at": str(r[10]),
+                }
+                for r in cur.fetchall()
+            ]
+
+    async def alist_runs(self, *args, **kwargs):
+        return await self.store.to_thread(self.list_runs, *args, **kwargs)
+
+    def list_results(self, run_id: str, limit: int = 100) -> list[dict]:
+        with self.store.connect() as conn:
+            cur = conn.cursor()
+            cur.execute(f"""
+                select JUDGE_NAME, JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE,
+                       ALUCINATION_SCORE, INFERRED_CSI_SCORE, RESOLUTION,
+                       CONVERSATION_PRECISION, RATIONALE, TRACE_ID, SESSION_ID, CREATED_AT
+                  from {self.store.t('EVALUATION_RESULT')}
+                 where RUN_ID = :run_id
+                 order by CREATED_AT desc
+            """, {"run_id": run_id})
+            return [
+                {
+                    "judge_name": r[0],
+                    "judge_type": r[1],
+                    "score": r[2],
+                    "judge_score": r[3],
+                    "accuracy_score": r[4],
+                    "alucination_score": r[5],
+                    "inferred_csi_score": r[6],
+                    "resolution": r[7],
+                    "conversation_precision": r[8],
+                    "rationale": r[9],
+                    "trace_id": r[10],
+                    "session_id": r[11],
+                    "created_at": str(r[12]),
+                }
+                for r in cur.fetchall()[:limit]
+            ]
+
+    async def alist_results(self, *args, **kwargs):
+        return await self.store.to_thread(self.list_results, *args, **kwargs)

evaluator/prompts/loader.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+import yaml
+from evaluator.config.settings import settings
+
+
+def load_prompt(path: str, key: str) -> str:
+    p = settings.path(path)
+    data = yaml.safe_load(p.read_text()) or {}
+    if key not in data:
+        raise KeyError(f"Prompt key {key!r} not found in {p}")
+    return str(data[key])

evaluator/publishers/langfuse_scores.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+import httpx
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationRecord, TraceJudgeResult
+
+class LangfuseScorePublisher:
+    async def publish_trace_score(self, record: ConversationRecord, result: TraceJudgeResult):
+        if not settings.can_publish_langfuse_scores or not record.trace_id:
+            return None
+        auth = (settings.langfuse_public_key, settings.langfuse_secret_key)
+        payloads = [
+            {'traceId': record.trace_id, 'name': 'offline_judge_score', 'value': result.judgeScore, 'comment': result.rationale},
+            {'traceId': record.trace_id, 'name': 'offline_accuracy_score', 'value': result.accuracyScore, 'comment': result.rationale},
+            {'traceId': record.trace_id, 'name': 'offline_alucination_score', 'value': result.alucinationScore, 'comment': result.rationale},
+        ]
+        async with httpx.AsyncClient(base_url=settings.langfuse_host, timeout=30) as client:
+            for payload in payloads:
+                resp = await client.post('/api/public/scores', json=payload, auth=auth)
+                if resp.status_code >= 400:
+                    # Don't fail the run because score publishing is supplementary.
+                    return {'ok': False, 'status': resp.status_code, 'body': resp.text}
+        return {'ok': True}

k8s/cronjob.yaml

@@ -0,0 +1,20 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: agent-framework-evaluator
+spec:
+  schedule: "0 2 * * *"
+  suspend: true
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+            - name: evaluator
+              image: agent-framework-evaluator:latest
+              command: ["python", "-m", "evaluator.cli", "run-agents", "--source", "langfuse"]
+              envFrom:
+                - secretRef:
+                    name: agent-framework-evaluator-env

pyproject.toml

@@ -0,0 +1,25 @@
+[project]
+name = "agent-framework-evaluator"
+version = "0.2.0"
+description = "Offline LLM-as-a-Judge evaluator for Agent Framework conversations"
+requires-python = ">=3.11"
+dependencies = [
+  "python-dotenv>=1.0.1",
+  "pydantic>=2.7.0",
+  "pydantic-settings>=2.2.1",
+  "oracledb>=2.4.0",
+  "httpx>=0.27.0",
+  "pyyaml>=6.0.1",
+  "typer>=0.12.3",
+  "click>=8.1.7",
+  "rich>=13.7.0",
+  "fastapi>=0.111.0",
+  "uvicorn>=0.30.0"
+]
+
+[project.scripts]
+af-evaluator = "evaluator.cli:app"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["evaluator*"]