commit 56a420f5f8a4330d2863141ae7d6dd740df61645 Author: Cristiano Hoshikawa Date: Fri Jun 12 11:36:33 2026 -0300 first commit diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..c3caf72 Binary files /dev/null and b/.DS_Store differ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..be9cadf --- /dev/null +++ b/.env.example @@ -0,0 +1,38 @@ +# Oracle Autonomous Database / Wallet - same pattern used by Agent Framework +ADB_USER=admin +ADB_PASSWORD=change-me +ADB_DSN=oradb23ai_high +ADB_WALLET_LOCATION=/path/to/Wallet_ORADB23ai +ADB_WALLET_PASSWORD=change-me +ADB_TABLE_PREFIX=AGENTFW + +# Langfuse collector / publisher +ENABLE_LANGFUSE=true +LANGFUSE_PUBLIC_KEY=pk-lf-... +LANGFUSE_SECRET_KEY=sk-lf-... +LANGFUSE_HOST=http://localhost:3005 +PUBLISH_LANGFUSE_SCORES=false + +# LLM - same style as Agent Framework env-driven config +LLM_PROVIDER=oci_openai +LLM_PROFILE=judge +LLM_PROFILES_PATH=configs/llm_profiles/llm_profiles.yaml +OCI_GENAI_BASE_URL=https://inference.generativeai.sa-saopaulo-1.oci.oraclecloud.com/openai/v1 +OCI_GENAI_MODEL_ID=meta.llama-3.3-70b-instruct +OCI_GENAI_API_KEY=seu_token_aqui +LLM_TEMPERATURE=0 +LLM_MAX_TOKENS=900 +LLM_TIMEOUT_SECONDS=120 + +# Execution +AGENTS_CONFIG_PATH=configs/judge/agents.yaml +TRACE_PROMPT_PATH=configs/judge/trace_metrics.yaml +SESSION_PROMPT_PATH=configs/judge/session_metrics.yaml +OUTPUT_DIR=output +BATCH_SIZE=50 +MAX_ATTEMPTS=3 + +# Optional GCS export compatibility with TIM package +ENABLE_GCS_UPLOAD=false +JUDGE_GCS_BUCKET= +GOOGLE_APPLICATION_CREDENTIALS=configs/GCP_ACCESS_KEY.json diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..7d9a8e5 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,12 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Environment-dependent path to Maven home directory +/mavenHomeManager.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Zeppelin ignored files +/ZeppelinRemoteNotebooks/ diff --git a/.idea/agent_framework_evaluator.iml b/.idea/agent_framework_evaluator.iml new file mode 100644 index 0000000..d6ebd48 --- /dev/null +++ b/.idea/agent_framework_evaluator.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..919ce1f --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..89ee753 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..589bacc --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.oca/custom_code_review_guidelines.txt b/.oca/custom_code_review_guidelines.txt new file mode 100644 index 0000000..a0a3b63 --- /dev/null +++ b/.oca/custom_code_review_guidelines.txt @@ -0,0 +1,24 @@ +# Sample guideline, please follow similar structure for guideline with code samples +# 1. Suggest using streams instead of simple loops for better readability. +# +# *Comment: +# Category: Minor +# Issue: Use streams instead of a loop for better readability. +# Code Block: +# +# ```java +# // Calculate squares of numbers +# List squares = new ArrayList<>(); +# for (int number : numbers) { +# squares.add(number * number); +# } +# ``` +# Recommendation: +# +# ```java +# // Calculate squares of numbers +# List squares = Arrays.stream(numbers) +# .map(n -> n * n) // Map each number to its square +# .toList(); +# ``` +# diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0c7addf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +WORKDIR /app +COPY . /app +RUN pip install --no-cache-dir -e . +CMD ["python", "-m", "evaluator.cli", "run-agents", "--source", "langfuse"] diff --git a/README.en-US.md b/README.en-US.md new file mode 100644 index 0000000..97ea9db --- /dev/null +++ b/README.en-US.md @@ -0,0 +1,1546 @@ +# agent_framework_evaluator + +## 1. What is the `agent_framework_evaluator`? + +The `agent_framework_evaluator` is a complementary service to the `agent_framework_oci` created to evaluate real conversations conducted by the framework's agents. + +It collects conversations from a source, usually Langfuse, reconstructs the context of the interaction, runs a Judge LLM, writes the results to an Oracle/ADB database, generates legacy files in TXT.GZ format, and optionally publishes scores back to Langfuse. + +In simple terms: + +```text +agent_framework_oci gera conversas e telemetria + ↓ +Langfuse armazena traces, spans, generations, metadata e usage + ↓ +agent_framework_evaluator coleta essas conversas + ↓ +LLM Judge avalia qualidade, precisão, alucinação, resolução e CSI + ↓ +Oracle/ADB persiste runs, itens, resultados, achados e progresso + ↓ +Exporter gera arquivo legado AGENTE__LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +The evaluator does not replace the guardrails, online judges, or telemetry of `agent_framework_oci`. It acts as an offline/batch layer for evaluation, auditing, and export. + +--- + +## 2. Purpose of the solution + +The purpose of the evaluator is to allow conversations that have already taken place to be analyzed later using standardized criteria. + +It mainly serves these scenarios: + +- daily evaluation of conversations by agent; +- generation of legacy evaluation files; +- auditing the quality of responses; +- identification of hallucination, low accuracy, low resolution or poor customer experience; +- comparison between agents such as `telecom_contas`, `retail_orders` and `financeiro_agent`; +- optional publication of scores on Langfuse; +- persistence of evaluation history in Oracle/ADB; +- progress tracking via API or CLI. + +--- + +## 3. How it integrates with `agent_framework_oci` + +`agent_framework_oci` is the main runtime for agents. It executes the conversational flow with LangGraph, supervisor, guardrails, judges, MCP tools, memory, RAG, and telemetry. + +During execution, the framework publishes traces to Langfuse containing: + +- `trace_id`; +- `session_id`; +- `message_id`; +- `agent_id`; +- `channel`; +- canonical `business_context`; +- IC/NOC/GRL events; +- LangGraph spans; +- guardrail spans; +- judge spans; +- LLM generations; +- model usage, when available; +- `prompt_tokens`, `completion_tokens` and `total_tokens`, when returned by the provider; +- `input_size`, when emitted by the framework spans. + +The evaluator uses this telemetry as a data source. + +The main integration happens like this: + +```text +agent_framework_oci +├── Executes agents +├── Resolves identity via identity.yaml +├── Creates canonical BusinessContext +├── Calls MCP/RAG/LLM +├── Emits Langfuse telemetry +└── Writes usage/model/tokens when available + +agent_framework_evaluator +├── Reads traces in Langfuse +├── Applies identity.yaml to normalize identity +├── Rebuilds ConversationRecord +├── Executes LLM Judge offline +├── Writes results to Oracle/ADB +├── Exports legacy TXT.GZ +└── Optionally publish scores on Langfuse +``` + +--- + +## 4. General architecture + +```text ++------------------------+ +| agent_framework_oci | +|------------------------| +| LangGraph | +| Supervisor | +| Guardrails | +| Judges online | +| MCP Tool Router | +| RAG | +| Memory / Checkpoint | +| Langfuse Telemetry | ++-----------+------------+ + | + v ++------------------------+ +| Langfuse | +|------------------------| +| Traces | +| Spans | +| Generations | +| Metadata | +| Usage / Tokens | ++-----------+------------+ + | + v ++------------------------+ +| agent_framework_ | +| evaluator | +|------------------------| +| Collectors | +| Identity Resolver | +| Conversation Records | +| LLM Judge | +| VLoop analytics | +| Repository Oracle | +| Legacy Exporter | +| API / CLI | ++-----------+------------+ + | + v ++------------------------+ +| Oracle ADB | +|------------------------| +| EVALUATION_RUN | +| EVALUATION_ITEM | +| EVALUATION_RESULT | +| EVALUATION_FINDING | +| EVALUATION_PROGRESS | +| EVALUATION_METRIC | ++-----------+------------+ + | + v ++------------------------+ +| Output | +|------------------------| +| TXT.GZ legacy | +| API dashboard | +| Langfuse scores | ++------------------------+ +``` + +--- + +## 5. Solution components + +### 5.1 CLI + +Main file: + +```text +evaluator/cli.py +``` + +Responsible for exposing commands such as: + +```bash +python -m evaluator.cli init-db +python -m evaluator.cli show-config +python -m evaluator.cli run --source langfuse +python -m evaluator.cli run-agents --source langfuse +python -m evaluator.cli runs +python -m evaluator.cli progress +``` + +The CLI is the main way to operate the evaluator in batch mode. + +--- + +### 5.2 API + +Main file: + +```text +evaluator/api/main.py +``` + +Exposes HTTP endpoints to query progress, runs, and results. + +Expected examples: + +```text +GET /health +GET /runs +GET /runs/{run_id}/progress +GET /runs/{run_id}/results +GET /runs/{run_id}/findings +``` + +The API allows you to build a simple graphical interface or integrate the evaluator with other systems. + +--- + +### 5.3 EvaluationEngine + +Main file: + +```text +evaluator/engine.py +``` + +It is the central orchestrator of the evaluator. + +Responsibilities: + +1. create a new evaluation run (`EVALUATION_RUN`); +2. choose the collector according to the `source`; +3. collect conversations; +4. apply sampling by agent; +5. insert items into `EVALUATION_ITEM`; +6. process each item; +7. call the LLM Judge; +8. save trace result; +9. run session evaluation; +10. save session result; +11. export legacy file; +12. mark final execution status; +13. issue progress events. + +Simplified flow: + +```text +run_agent() + ↓ +collector.collect() + ↓ +repository.insert_items() + ↓ +_process() + ↓ +judge.judge_trace() + ↓ +repository.save_trace_result() + ↓ +judge.judge_sessions() + ↓ +repository.save_session_result() + ↓ +export_legacy_txt_gz() +``` + +--- + +### 5.4 Collectors + +Directory: + +```text +evaluator/collectors/ +``` + +Collectors are responsible for fetching conversations from an external source and converting them to `ConversationRecord`. + +Typical collectors: + +```text +evaluator/collectors/langfuse.py +evaluator/collectors/agent_framework.py +evaluator/collectors/mock.py +evaluator/collectors/base.py +``` + +#### LangfuseCollector + +This is the main collector. + +Responsibilities: + +- search for traces in Langfuse; +- filter by period; +- filter by agent/alias; +- retrieve trace details; +- extract input/output; +- reconstruct messages; +- collect metadata; +- apply `identity.yaml`; +- assemble canonical `BusinessContext`; +- fill in `ConversationRecord`. + +The collector must normalize data so that the exporter does not need to know Langfuse's internal details. + +--- + +### 5.5 Identity Resolver + +Recommended directory: + +```text +evaluator/identity/ +``` + +Main file: + +```text +evaluator/identity/resolver.py +``` + +The evaluator must use the same identity concept as `agent_framework_oci`, based on the file: + +```text +configs/identity.yaml +``` + +The function of `identity.yaml` is to map variable input fields to a canonical model: + +```text +customer_key +contract_key +interaction_key +account_key +resource_key +session_key +``` + +Conceptual example: + +```yaml +identity: + version: 2 + keys: + customer_key: + sources: + - business_context.customer_key + - metadata.customer_key + - user_id + contract_key: + sources: + - business_context.contract_key + - metadata.contract_key + interaction_key: + sources: + - business_context.interaction_key + - metadata.ura_call_id + - metadata.message_id + - message_id + session_key: + sources: + - business_context.session_key + - session_id + - conversation_key +``` + +With this, the evaluator is not directly tied to fields such as `ura_call_id`, `call_id`, `message_id` or `interaction_key`. It resolves everything to `interaction_key`. + +--- + +### 5.6 Models + +Main file: + +```text +evaluator/core/models.py +``` + +Defines the core objects of the evaluator. + +Main models: + +```python +class ConversationRecord +class ConversationMessage +class TraceJudgeResult +class SessionJudgeResult +class CombinedJudgeResult +class RunStatus +class ItemStatus +``` + +#### ConversationRecord + +Represents an evaluated conversation or turn. + +Common fields: + +```text +trace_id +session_id +message_id +agent_id +channel +input_text +output_text +messages +metadata +raw +``` + +The `metadata` field must contain normalized data: + +```text +business_context +uraCallId +channelId +messageId +promptLength +``` + +The `raw` field keeps the original payload for auditing and fallback. + +--- + +### 5.7 LLM Judge + +Main file: + +```text +evaluator/judges/llm_judge.py +``` + +Main class: + +```python +TIMStyleLLMJudge +``` + +Responsibilities: + +- load evaluation prompts; +- set up trace prompt; +- set up session prompt; +- call LLM via configured client; +- interpret JSON response; +- return `TraceJudgeResult` and `SessionJudgeResult`. + +The judge evaluates metrics such as: + +```text +judgeScore +accuracyScore +alucinationScore +inferredCsiScore +resolution +conversationPrecision +rationale +``` + +The judge must be LLM-based, not deterministic. + +--- + +### 5.8 Prompts + +Directory: + +```text +evaluator/prompts/ +``` + +Expected files: + +```text +trace_judge_prompt.md +session_judge_prompt.md +loader.py +``` + +The trace prompt evaluates an individual response. + +The session prompt evaluates the conversation grouped by `session_id`. + +Example of expected LLM output for trace: + +```json +{ + "judgeScore": 0.8, + "accuracyScore": 0.9, + "alucinationScore": 0.1, + "rationale": "A response that is relevant to the context and based on available data." +} +``` + +Example of expected output for session: + +```json +{ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "The conversation was resolved with consistent information." +} +``` + +--- + +### 5.9 LLM Client + +Directory: + +```text +evaluator/llm/ +``` + +Typical files: + +```text +evaluator/llm/client.py +evaluator/llm/oci_openai.py +``` + +The evaluator must use the same LLM access pattern as `agent_framework_oci`, preferably via the `oci_openai` provider. + +Common variables: + +```env +LLM_PROVIDER=oci_openai +OCI_GENAI_ENDPOINT=... +OCI_GENAI_MODEL_ID=... +OCI_GENAI_API_KEY=... +OCI_GENAI_COMPARTMENT_ID=... +``` + +The client needs to return raw text for the Judge to interpret as JSON. + +--- + +### 5.10 Repository / Oracle Store + +Directory: + +```text +evaluator/persistence/ +``` + +Main files: + +```text +evaluator/persistence/oracle_store.py +evaluator/persistence/repository.py +``` + +`OracleStore` takes care of: + +- connection with ADB/Oracle; +- wallet; +- DSN; +- schema creation/adjustment; +- thread-safe execution for asynchronous calls; +- table prefix. + +The `EvaluationRepository` takes care of: + +- creating runs; +- recording progress; +- inserting items; +- search for next items; +- marking an item as `PROCESSING`, `COMPLETED` or `FAILED`; +- save results; +- save findings; +- summarize run; +- list runs; +- check progress. + +--- + +### 5.11 Legacy Exporter + +Main file: + +```text +evaluator/output/legacy_exporter.py +``` + +Generates the legacy file: + +```text +output/AGENTE__LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +Column format: + +```text +judgeScore +accuracyScore +alucinationScore +promptLength +loop +inferredCsiScore +resolution +conversationPrecision +uraCallId +channelId +sessionId +messageId +``` + +Example: + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"TOTAL"|;"19" +``` + +#### promptLength + +The `promptLength` field must use this priority: + +1. `prompt_tokens`/ `promptTokens` /`input_tokens`/ `inputTokens` in Langfuse observations; +2. `usage.input` or `usageDetails.input`; +3. `metadata.input_size` issued by the framework; +4. fallback for text size of `input_text`, `output_text`, and `messages`. + +Example: + +```text +promptLength = 732 +``` + +#### loop + +The `loop field` uses the VLoop detector. + +```text +0 = sem loop detectado +1 = loop detectado +``` + +--- + +### 5.12 VLoop Analytics + +Main file: + +```text +evaluator/analytics/vloop.py +``` + +Responsible for detecting conversational repetition/loop in a pattern similar to the VLoop guardrail of `agent_framework_oci`. + +The function normally exposed is: + +```python +vloop_flag(raw) -> int +``` + +It returns: + +```text +0 when there is no evidence of a loop +1 when there is suspected repetition +``` + +--- + +### 5.13 Langfuse Score Publisher + +Main file: + +```text +evaluator/publishers/langfuse_scores.py +``` + +Responsible for publishing evaluation scores back to Langfuse, when enabled. + +Control variable: + +```env +PUBLISH_LANGFUSE_SCORES=true +``` + +When disabled, the evaluator only writes to the database and exports the file. + +--- + +## 6. Directory structure + +```text +agent_framework_evaluator/ +├── configs/ +│ ├── identity.yaml +│ └── judge/ +│ └── agents.yaml +├── docs/ +├── evaluator/ +│ ├── __init__.py +│ ├── cli.py +│ ├── engine.py +│ ├── api/ +│ │ └── main.py +│ ├── analytics/ +│ │ └── vloop.py +│ ├── collectors/ +│ │ ├── base.py +│ │ ├── langfuse.py +│ │ ├── agent_framework.py +│ │ └── mock.py +│ ├── config/ +│ │ ├── settings.py +│ │ └── agents.py +│ ├── core/ +│ │ └── models.py +│ ├── identity/ +│ │ └── resolver.py +│ ├── judges/ +│ │ └── llm_judge.py +│ ├── llm/ +│ │ ├── client.py +│ │ └── oci_openai.py +│ ├── output/ +│ │ └── legacy_exporter.py +│ ├── persistence/ +│ │ ├── oracle_store.py +│ │ └── repository.py +│ ├── prompts/ +│ │ ├── loader.py +│ │ ├── trace_judge_prompt.md +│ │ └── session_judge_prompt.md +│ └── publishers/ +│ └── langfuse_scores.py +├── output/ +├── Dockerfile +├── docker-compose.yml +├── pyproject.toml +└── README.md +``` + +--- + +## 7. Configuration + +### 7.1 `.env file` + +Example: + +```env +# Oracle / ADB +ADB_USER=ADMIN +ADB_PASSWORD=your_password +ADB_DSN=oradb23ai_high +ADB_WALLET_DIR=/path/to/Wallet_ORADB23ai +DB_TABLE_PREFIX=AGENTFW_ + +# Langfuse +LANGFUSE_ENABLED=true +LANGFUSE_HOST=http://localhost:3005 +LANGFUSE_PUBLIC_KEY=pk-lf-... +LANGFUSE_SECRET_KEY=sk-lf-... +PUBLISH_LANGFUSE_SCORES=false + +# LLM +LLM_PROVIDER=oci_openai +OCI_GENAI_ENDPOINT=https://... +OCI_GENAI_MODEL_ID=... +OCI_GENAI_API_KEY=... +OCI_GENAI_COMPARTMENT_ID=... + +# Evaluator +EVALUATOR_OUTPUT_DIR=output +EVALUATOR_BATCH_SIZE=10 +EVALUATOR_MAX_ATTEMPTS=2 +EVALUATOR_AGENTS_CONFIG=configs/judge/agents.yaml +IDENTITY_CONFIG_PATH=configs/identity.yaml +TRACE_PROMPT_PATH=evaluator/prompts/trace_judge_prompt.md +SESSION_PROMPT_PATH=evaluator/prompts/session_judge_prompt.md +``` + +--- + +### 7.2 Agent configuration + +File: + +```text +configs/judge/agents.yaml +``` + +Example: + +```yaml +agents: + - agent_id: telecom_contas + enabled: true + aliases: + - telecom_contas + - billing_agent + - financeiro_agent + percentage: 1.0 + + - agent_id: retail_orders + enabled: true + aliases: + - retail_orders + - orders_agent + percentage: 1.0 + + - agent_id: financeiro_agent + enabled: true + aliases: + - financeiro_agent + percentage: 1.0 +``` + +The `aliases` field is important because Langfuse can register the agent in different ways, for example: + +```text +agent_id = telecom_contas +route = financeiro_agent +agent = financeiro_agent +``` + +--- + +### 7.3 Identity configuration + +File: + +```text +configs/identity.yaml +``` + +The evaluator must use the same pattern as the framework. + +Example: + +```yaml +identity: + version: 2 + keys: + customer_key: + sources: + - business_context.customer_key + - metadata.customer_key + - user_id + + contract_key: + sources: + - business_context.contract_key + - metadata.contract_key + + interaction_key: + sources: + - business_context.interaction_key + - metadata.ura_call_id + - metadata.message_id + - message_id + + session_key: + sources: + - business_context.session_key + - metadata.session_key + - session_id + - conversation_key +``` + +The `interaction_key` field is used to populate the `uraCallId` in the legacy export. + +--- + +## 8. How to run + +### 8.1 Install dependencies + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +If you are using Conda: + +```bash +conda activate py313 +pip install -e . +``` + +--- + +### 8.2 Validate configuration + +```bash +python -m evaluator.cli show-config +``` + +Expected output: + +```text +{ + "env_path": ".../.env", + "adb_dsn": "oradb23ai_high", + "wallet": ".../Wallet_ORADB23ai", + "langfuse": true, + "publish_langfuse_scores": false, + "llm_provider": "oci_openai", + "agents_config": "configs/judge/agents.yaml" +} +``` + +--- + +### 8.3 Create/validate schema + +```bash +python -m evaluator.cli init-db +``` + +Expected output: + +```text +{'status': 'OK', 'message': 'Evaluator schema checked/created successfully.'} +``` + +--- + +### 8.4 Run evaluation by period + +```bash +python -m evaluator.cli run \ + --period-start 2026-06-11T00:00:00 \ + --period-end 2026-06-12T00:00:00 \ + --source langfuse +``` + +--- + +### 8.5 Run evaluation by configured agents + +```bash +python -m evaluator.cli run-agents --source langfuse +``` + +Expected output: + +```text +[ + { + 'status': 'COMPLETED', + 'run_id': '...', + 'total_items': 19, + 'completed_items': 19, + 'failed_items': 0, + 'evaluations': 19, + 'avg_score': 0.72, + 'agent_id': 'telecom_contas', + 'output_file': 'output/AGENTE_telecom_contas_LLM_JUDGE_20260612.TXT.GZ', + 'uploaded_to': None + } +] +``` + +--- + +### 8.6 Check progress + +```bash +python -m evaluator.cli progress +``` + +Or via API: + +```bash +curl http://localhost:8001/runs//progress +``` + +--- + +### 8.7 View exported file + +```bash +gzip -cd output/AGENTE_telecom_contas_LLM_JUDGE_20260612.TXT.GZ +``` + +Example of a valid line: + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"TOTAL"|;"19" +``` + +--- + +## 9. Database + +### 9.1 Main tables + +#### EVALUATION_RUN + +Stores an evaluation run. + +Main fields: + +```text +RUN_ID +PERIOD_START +PERIOD_END +SOURCE +AGENT_ID +STATUS +TOTAL_ITEMS +PROCESSED_ITEMS +FAILED_ITEMS +LAST_HEARTBEAT_AT +CREATED_AT +UPDATED_AT +ERROR_MESSAGE +``` + +--- + +#### EVALUATION_ITEM + +Stores each conversation/turn collected. + +Main fields: + +```text +ITEM_ID +RUN_ID +TRACE_ID +SESSION_ID +MESSAGE_ID +AGENT_ID +CHANNEL +STATUS +ATTEMPT_COUNT +RAW_JSON +CREATED_AT +UPDATED_AT +ERROR_MESSAGE +``` + +--- + +#### EVALUATION_RESULT + +Stores trace and session results. + +Main fields: + +```text +RESULT_ID +RUN_ID +ITEM_ID +TRACE_ID +SESSION_ID +AGENT_ID +JUDGE_TYPE +JUDGE_NAME +JUDGE_SCORE +ACCURACY_SCORE +ALUCINATION_SCORE +INFERRED_CSI_SCORE +RESOLUTION +CONVERSATION_PRECISION +RATIONALE +RESULT_JSON +CREATED_AT +``` + +`JUDGE_TYPE` can be: + +```text +TRACE +SESSION +``` + +--- + +#### EVALUATION_PROGRESS_EVENT + +Stores execution progress events. + +Stage examples: + +```text +RUN_CREATED +COLLECTING +COLLECTED +SAMPLED +ITEMS_INSERTED +BATCH_STARTED +ITEM_COMPLETED +ITEM_FAILED +SESSION_JUDGE_COMPLETED +EXPORTED +COMPLETED +PARTIAL +``` + +--- + +## 10. How the codes work together + +### 10.1 Complete execution flow + +```text +CLI run-agents + ↓ +load configs/judge/agents.yaml + ↓ +for each enabled agent + ↓ +EvaluationEngine.run_agent(agent) + ↓ +cria EVALUATION_RUN + ↓ +LangfuseCollector.collect(...) + ↓ +IdentityResolver.resolve(...) + ↓ +ConversationRecord + ↓ +EvaluationRepository.insert_items(...) + ↓ +EvaluationEngine._process(run_id) + ↓ +TIMStyleLLMJudge.judge_trace(record) + ↓ +LLMClient.complete(prompt) + ↓ +save_trace_result(...) + ↓ +TIMStyleLLMJudge.judge_sessions(records) + ↓ +save_session_result(...) + ↓ +export_legacy_txt_gz(...) + ↓ +COMPLETED +``` + +--- + +### 10.2 Role of the collector + +The collector is responsible for transforming external data into canonical data. + +It must hide differences between sources such as: + +```text +Langfuse +agent_framework database +mock data +``` + +The output must always be: + +```python +ConversationRecord +``` + +--- + +### 10.3 Role of the judge + +The judge receives a `ConversationRecord`, assembles a prompt, and calls the LLM. + +It should not know about Oracle, Langfuse, legacy export, or API. + +It only evaluates. + +--- + +### 10.4 Role of the repository + +The repository is the persistence layer. + +It must not contain an evaluation business rule. + +It only writes, retrieves, and updates data. + +--- + +### 10.5 Role of the exporter + +The exporter transforms persisted data into a legacy file. + +It should not resolve identity in a complex way. + +Ideally, it should read fields that are already normalized: + +```text +metadata.business_context.interaction_key +metadata.channelId +metadata.messageId +metadata.promptLength +``` + +However, for resilience, it can also query `RAW_JSON` as a fallback. + +--- + +## 11. Important design rules + +### 11.1 The evaluator must not be anchored to an agent + +Avoid logic like: + +```python +if agent_id == "telecom_contas": + ura_call_id = metadata["ura_call_id"] +``` + +The correct thing to do is to use `identity.yaml`. + +--- + +### 11.2 The exporter must not know internal details of Langfuse + +Avoid excessive coupling to paths such as: + +```text +raw.detail.observations[0].metadata.ura_call_id +raw.trace.input.business_context.interaction_key +``` + +This should be resolved in the collector. + +--- + +### 11.3 `promptLength` should come from tokens when possible + +Recommended priority: + +```text +1. prompt_tokens / promptTokens +2. input_tokens / inputTokens +3. usage.input / usageDetails.input +4. metadata.input_size +5. tamanho textual de input/output/messages +``` + +--- + +### 11.4 `uraCallId` must come from BusinessContext + +The legacy field `uraCallId` must be mapped to: + +```text +business_context.interaction_key +``` + +This is the canonical name of the framework. + +--- + +### 11.5 `sessionId` must come from BusinessContext + +The legacy `sessionId` field must be mapped to: + +```text +business_context.session_key +``` + +Not to be confused with the full composite key: + +```text +default:telecom_contas: +``` + +The evaluator can store the full key, but the legacy export should normally use the clean session identifier. + +--- + +## 12. Recommended tests + +### 12.1 Configuration test + +```bash +python -m evaluator.cli show-config +``` + +Validate: + +```text +ADB_DSN +Wallet +Langfuse enabled +LLM provider +Agents config +Identity config +``` + +--- + +### 12.2 Database test + +```bash +python -m evaluator.cli init-db +``` + +Then validate tables: + +```sql +select table_name +from user_tables +where table_name like 'AGENTFW_EVALUATION%'; +``` + +--- + +### 12.3 Mock test + +```bash +python -m evaluator.cli run --source mock +``` + +Use this test to validate schema, judge, and export without relying on Langfuse. + +--- + +### 12.4 Test with Langfuse + +```bash +python -m evaluator.cli run-agents --source langfuse +``` + +Validate: + +```text +total_items > 0 +completed_items > 0 +failed_items = 0 +evaluations > 0 +output_file preenchido +``` + +--- + +### 12.5 Export test + +```bash +gzip -cd output/AGENTE_telecom_contas_LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +Validate columns: + +```text +judgeScore filled in +accuracyScore filled in +hallucinationScore filled in +promptLength greater than 0 +loop 0 or 1 +inferredCsiScore filled in +resolution 0 or 1 +conversationPrecision 0 or 1 +uraCallId filled in +channelId filled in +sessionId filled in +messageId filled in +``` + +--- + +## 13. Troubleshooting + +### 13.1 `promptLength` outputs 0 + +Common causes: + +- `find_prompt_tokens` was not included in the file; +- `promptTokens` is zeroed in Langfuse; +- `input_size` is not being traversed; +- `RAW_JSON` is coming as an unconverted string; +- old exporter is still running; +- `except Exception: pass` is masking error. + +Recommended debug: + +```python +print("PROMPT_LENGTH", extract_prompt_length(raw)) +print("RAW_TYPE", type(raw)) +print("RAW_KEYS", list(raw.keys())[:20]) +``` + +--- + +### 13.2 `uraCallId` comes out empty + +Common causes: + +- `identity.yaml` is not being loaded; +- collector is not copying `business_context` to `metadata`; +- `interaction_key` does not exist in the trace; +- exporter does not use `business_context.interaction_key`. + +Validation: + +```sql +select RAW_JSON +from AGENTFW_EVALUATION_ITEM +where MESSAGE_ID = ''; +``` + +Search: + +```text +interaction_key +ura_call_id +business_context +``` + +--- + +### 13.3 `ORA-00904 invalid identifier` + +Usually indicates an old schema. + +Examples already found: + +```text +ORA-00904: UPDATED_AT invalid identifier +ORA-00904: REASONING invalid identifier +ORA-00904: JUDGE_TYPE invalid identifier +``` + +Correction: + +```bash +python -m evaluator.cli init-db +``` + +If the table already exists without the new column,`_init_schema` needs to run `ALTER TABLE ADD` in an idempotent manner. + +--- + +### 13.4 `ORA-00054 resource busy` + +Indicates a lock on the table. + +Common causes: + +- API running while `init-db` tries to change schema; +- another process using the table; +- transaction open in SQL Developer. + +Correction: + +1. stop API/CLI; +2. close open sessions; +3. run `init-db` again. + +--- + +### 13.5 `OCI LLM 401` + +Indicates an authentication problem in the LLM. + +Validate: + +```env +OCI_GENAI_ENDPOINT +OCI_GENAI_MODEL_ID +OCI_GENAI_API_KEY +OCI_GENAI_COMPARTMENT_ID +``` + +Also confirm that the evaluator is reading the correct `.env`: + +```bash +python -m evaluator.cli show-config +``` + +--- + +### 13.6 `Entity with key ${OCI_GENAI_MODEL_ID} not found` + +Indicates that the literal value `${OCI_GENAI_MODEL_ID}` has reached the provider. + +Common causes: + +- variable not expanded; +- YAML using `${OCI_GENAI_MODEL_ID}` without interpolation; +- `.env` not loaded; +- LLM client configuration does not resolve placeholders. + +Correction: + +- put the real model ID in `the .env`; +- ensure interpolation in `settings.py`; +- validate with `show-config`. + +--- + +## 14. Final validation checklist + +Before considering the evaluator ready, validate: + +```text +[ ] init-db executes without error +[ ] show-config displays correct .env file +[ ] Langfuse returns traces +[ ] run-agents collects items per agent +[ ] LLM Judge responds with valid JSON +[ ] EVALUATION_RESULT records TRACE and SESSION data +[ ] progress displays useful events +[ ] export TXT.GZ is generated +[ ] promptLength > 0 +[ ] uraCallId populated +[ ] sessionId populated +[ ] messageId populated +[ ] loop populated with 0 or 1 +[ ] file ends with TOTAL +[ ] scores can be published to Langfuse when enabled +``` + +--- + +## 15. Example of validated final result + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"0.9"|;"1"|;"0"|;"642"|;"0"|;"0.5"|;"1"|;"1"|;"5ab3ea80-7428-402f-98ec-04e7cd5327e4"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"5ab3ea80-7428-402f-98ec-04e7cd5327e4" +"TOTAL"|;"19" +``` + +This result indicates: + +- Judge working; +- prompt tokens extracted correctly; +- VLoop without occurrence; +- session metrics filled in; +- canonical identity working; +- legacy export in the expected layout. + +--- + +## 16. Executive summary + +The `agent_framework_evaluator` is the batch/offline evaluation layer of the `agent_framework_oci` ecosystem. + +It consumes the telemetry generated by the framework, applies a Judge LLM with evaluation rules, persists results in Oracle/ADB, generates a file, and can republish scores in Langfuse. + +The correct architecture separates responsibilities: + +```text +Collector normalizes data. +IdentityResolver resolves identity. +Judge evaluates conversation. +Repository persists data. +Exporter generates legacy data. +API/CLI operate the solution. +``` + +This makes the evaluator generic for multiple agents and avoids direct coupling to specific trace or payload formats. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b61a9dc --- /dev/null +++ b/README.md @@ -0,0 +1,1546 @@ +# agent_framework_evaluator + +## 1. O que é o `agent_framework_evaluator`? + +O `agent_framework_evaluator` é um serviço complementar ao `agent_framework_oci` criado para avaliar conversas reais executadas pelos agentes do framework. + +Ele coleta conversas de uma fonte, normalmente o Langfuse, reconstrói o contexto da interação, executa um Judge LLM, grava os resultados em banco Oracle/ADB, gera arquivos legados no formato TXT.GZ e, opcionalmente, publica scores de volta no Langfuse. + +Em termos simples: + +```text +agent_framework_oci gera conversas e telemetria + ↓ +Langfuse armazena traces, spans, generations, metadata e usage + ↓ +agent_framework_evaluator coleta essas conversas + ↓ +LLM Judge avalia qualidade, precisão, alucinação, resolução e CSI + ↓ +Oracle/ADB persiste runs, itens, resultados, achados e progresso + ↓ +Exporter gera arquivo legado AGENTE__LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +O evaluator não substitui os guardrails, judges online ou telemetria do `agent_framework_oci`. Ele atua como uma camada offline/batch de avaliação, auditoria e exportação. + +--- + +## 2. Objetivo da solução + +O objetivo do evaluator é permitir que conversas já executadas sejam analisadas posteriormente com critérios padronizados. + +Ele atende principalmente estes cenários: + +- avaliação diária de conversas por agente; +- geração de arquivos legados de avaliação; +- auditoria de qualidade de respostas; +- identificação de alucinação, baixa precisão, baixa resolução ou baixa experiência do cliente; +- comparação entre agentes como `telecom_contas`, `retail_orders` e `financeiro_agent`; +- publicação opcional de scores no Langfuse; +- persistência de histórico de avaliações no Oracle/ADB; +- acompanhamento de progresso via API ou CLI. + +--- + +## 3. Como ele se integra ao `agent_framework_oci` + +O `agent_framework_oci` é o runtime principal dos agentes. Ele executa o fluxo conversacional com LangGraph, supervisor, guardrails, judges, MCP tools, memória, RAG e telemetria. + +Durante a execução, o framework publica traces no Langfuse contendo: + +- `trace_id`; +- `session_id`; +- `message_id`; +- `agent_id`; +- `channel`; +- `business_context` canônico; +- eventos IC/NOC/GRL; +- spans de LangGraph; +- spans de guardrails; +- spans de judges; +- generations LLM; +- usage de modelo, quando disponível; +- `prompt_tokens`, `completion_tokens` e `total_tokens`, quando retornados pelo provider; +- `input_size`, quando emitido pelos spans do framework. + +O evaluator usa essa telemetria como fonte de dados. + +A integração principal acontece assim: + +```text +agent_framework_oci + ├── executa agentes + ├── resolve identidade via identity.yaml + ├── monta BusinessContext canônico + ├── chama MCP/RAG/LLM + ├── emite telemetria Langfuse + └── grava usage/model/tokens quando disponíveis + +agent_framework_evaluator + ├── lê traces no Langfuse + ├── aplica identity.yaml para normalizar identidade + ├── reconstrói ConversationRecord + ├── executa LLM Judge offline + ├── grava resultados no Oracle/ADB + ├── exporta TXT.GZ legado + └── opcionalmente publica scores no Langfuse +``` + +--- + +## 4. Arquitetura geral + +```text ++------------------------+ +| agent_framework_oci | +|------------------------| +| LangGraph | +| Supervisor | +| Guardrails | +| Judges online | +| MCP Tool Router | +| RAG | +| Memory / Checkpoint | +| Langfuse Telemetry | ++-----------+------------+ + | + v ++------------------------+ +| Langfuse | +|------------------------| +| Traces | +| Spans | +| Generations | +| Metadata | +| Usage / Tokens | ++-----------+------------+ + | + v ++------------------------+ +| agent_framework_ | +| evaluator | +|------------------------| +| Collectors | +| Identity Resolver | +| Conversation Records | +| LLM Judge | +| VLoop analytics | +| Repository Oracle | +| Legacy Exporter | +| API / CLI | ++-----------+------------+ + | + v ++------------------------+ +| Oracle ADB | +|------------------------| +| EVALUATION_RUN | +| EVALUATION_ITEM | +| EVALUATION_RESULT | +| EVALUATION_FINDING | +| EVALUATION_PROGRESS | +| EVALUATION_METRIC | ++-----------+------------+ + | + v ++------------------------+ +| Output | +|------------------------| +| TXT.GZ legado | +| API dashboard | +| Langfuse scores | ++------------------------+ +``` + +--- + +## 5. Componentes da solução + +### 5.1 CLI + +Arquivo principal: + +```text +evaluator/cli.py +``` + +Responsável por expor comandos como: + +```bash +python -m evaluator.cli init-db +python -m evaluator.cli show-config +python -m evaluator.cli run --source langfuse +python -m evaluator.cli run-agents --source langfuse +python -m evaluator.cli runs +python -m evaluator.cli progress +``` + +A CLI é a forma principal de operar o evaluator em modo batch. + +--- + +### 5.2 API + +Arquivo principal: + +```text +evaluator/api/main.py +``` + +Expõe endpoints HTTP para consultar progresso, runs e resultados. + +Exemplos esperados: + +```text +GET /health +GET /runs +GET /runs/{run_id}/progress +GET /runs/{run_id}/results +GET /runs/{run_id}/findings +``` + +A API permite construir uma interface gráfica simples ou integrar o evaluator com outros sistemas. + +--- + +### 5.3 EvaluationEngine + +Arquivo principal: + +```text +evaluator/engine.py +``` + +É o orquestrador central do evaluator. + +Responsabilidades: + +1. criar uma nova execução de avaliação (`EVALUATION_RUN`); +2. escolher o collector conforme `source`; +3. coletar conversas; +4. aplicar amostragem por agente; +5. inserir itens em `EVALUATION_ITEM`; +6. processar cada item; +7. chamar o LLM Judge; +8. salvar resultado de trace; +9. executar avaliação de sessão; +10. salvar resultado de sessão; +11. exportar arquivo legado; +12. marcar status final da execução; +13. emitir eventos de progresso. + +Fluxo simplificado: + +```text +run_agent() + ↓ +collector.collect() + ↓ +repository.insert_items() + ↓ +_process() + ↓ +judge.judge_trace() + ↓ +repository.save_trace_result() + ↓ +judge.judge_sessions() + ↓ +repository.save_session_result() + ↓ +export_legacy_txt_gz() +``` + +--- + +### 5.4 Collectors + +Diretório: + +```text +evaluator/collectors/ +``` + +Collectors são responsáveis por buscar conversas em uma fonte externa e convertê-las para `ConversationRecord`. + +Collectors típicos: + +```text +evaluator/collectors/langfuse.py +evaluator/collectors/agent_framework.py +evaluator/collectors/mock.py +evaluator/collectors/base.py +``` + +#### LangfuseCollector + +É o collector principal. + +Responsabilidades: + +- buscar traces no Langfuse; +- filtrar por período; +- filtrar por agente/alias; +- recuperar detalhes do trace; +- extrair input/output; +- reconstruir mensagens; +- coletar metadata; +- aplicar `identity.yaml`; +- montar `BusinessContext` canônico; +- preencher `ConversationRecord`. + +O collector deve normalizar dados para que o exporter não precise conhecer detalhes internos do Langfuse. + +--- + +### 5.5 Identity Resolver + +Diretório recomendado: + +```text +evaluator/identity/ +``` + +Arquivo principal: + +```text +evaluator/identity/resolver.py +``` + +O evaluator deve usar o mesmo conceito de identidade do `agent_framework_oci`, baseado no arquivo: + +```text +configs/identity.yaml +``` + +A função do `identity.yaml` é mapear campos variáveis de entrada para um modelo canônico: + +```text +customer_key +contract_key +interaction_key +account_key +resource_key +session_key +``` + +Exemplo conceitual: + +```yaml +identity: + version: 2 + keys: + customer_key: + sources: + - business_context.customer_key + - metadata.customer_key + - user_id + contract_key: + sources: + - business_context.contract_key + - metadata.contract_key + interaction_key: + sources: + - business_context.interaction_key + - metadata.ura_call_id + - metadata.message_id + - message_id + session_key: + sources: + - business_context.session_key + - session_id + - conversation_key +``` + +Com isso, o evaluator não fica preso a campos como `ura_call_id`, `call_id`, `message_id` ou `interaction_key` diretamente. Ele resolve tudo para `interaction_key`. + +--- + +### 5.6 Models + +Arquivo principal: + +```text +evaluator/core/models.py +``` + +Define os objetos centrais do evaluator. + +Principais modelos: + +```python +class ConversationRecord +class ConversationMessage +class TraceJudgeResult +class SessionJudgeResult +class CombinedJudgeResult +class RunStatus +class ItemStatus +``` + +#### ConversationRecord + +Representa uma conversa ou turno avaliado. + +Campos comuns: + +```text +trace_id +session_id +message_id +agent_id +channel +input_text +output_text +messages +metadata +raw +``` + +O campo `metadata` deve conter dados normalizados: + +```text +business_context +uraCallId +channelId +messageId +promptLength +``` + +O campo `raw` mantém o payload original para auditoria e fallback. + +--- + +### 5.7 LLM Judge + +Arquivo principal: + +```text +evaluator/judges/llm_judge.py +``` + +Classe principal: + +```python +TIMStyleLLMJudge +``` + +Responsabilidades: + +- carregar prompts de avaliação; +- montar prompt de trace; +- montar prompt de sessão; +- chamar LLM via client configurado; +- interpretar resposta JSON; +- retornar `TraceJudgeResult` e `SessionJudgeResult`. + +O judge avalia métricas como: + +```text +judgeScore +accuracyScore +alucinationScore +inferredCsiScore +resolution +conversationPrecision +rationale +``` + +O judge deve ser LLM-based, não determinístico. + +--- + +### 5.8 Prompts + +Diretório: + +```text +evaluator/prompts/ +``` + +Arquivos esperados: + +```text +trace_judge_prompt.md +session_judge_prompt.md +loader.py +``` + +O prompt de trace avalia uma resposta individual. + +O prompt de sessão avalia a conversa agrupada por `session_id`. + +Exemplo de saída esperada do LLM para trace: + +```json +{ + "judgeScore": 0.8, + "accuracyScore": 0.9, + "alucinationScore": 0.1, + "rationale": "Resposta aderente ao contexto e baseada em dados disponíveis." +} +``` + +Exemplo de saída esperada para sessão: + +```json +{ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "A conversa foi resolvida com informações consistentes." +} +``` + +--- + +### 5.9 LLM Client + +Diretório: + +```text +evaluator/llm/ +``` + +Arquivos típicos: + +```text +evaluator/llm/client.py +evaluator/llm/oci_openai.py +``` + +O evaluator deve usar o mesmo padrão de acesso a LLM do `agent_framework_oci`, preferencialmente via provider `oci_openai`. + +Variáveis comuns: + +```env +LLM_PROVIDER=oci_openai +OCI_GENAI_ENDPOINT=... +OCI_GENAI_MODEL_ID=... +OCI_GENAI_API_KEY=... +OCI_GENAI_COMPARTMENT_ID=... +``` + +O client precisa retornar texto bruto para o Judge interpretar como JSON. + +--- + +### 5.10 Repository / Oracle Store + +Diretório: + +```text +evaluator/persistence/ +``` + +Arquivos principais: + +```text +evaluator/persistence/oracle_store.py +evaluator/persistence/repository.py +``` + +O `OracleStore` cuida de: + +- conexão com ADB/Oracle; +- wallet; +- DSN; +- criação/ajuste de schema; +- execução thread-safe para chamadas assíncronas; +- prefixo de tabelas. + +O `EvaluationRepository` cuida de: + +- criar runs; +- gravar progresso; +- inserir itens; +- buscar próximos itens; +- marcar item como `PROCESSING`, `COMPLETED` ou `FAILED`; +- salvar resultados; +- salvar findings; +- sumarizar run; +- listar runs; +- consultar progresso. + +--- + +### 5.11 Legacy Exporter + +Arquivo principal: + +```text +evaluator/output/legacy_exporter.py +``` + +Gera o arquivo legado: + +```text +output/AGENTE__LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +Formato das colunas: + +```text +judgeScore +accuracyScore +alucinationScore +promptLength +loop +inferredCsiScore +resolution +conversationPrecision +uraCallId +channelId +sessionId +messageId +``` + +Exemplo: + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"TOTAL"|;"19" +``` + +#### promptLength + +O campo `promptLength` deve usar esta prioridade: + +1. `prompt_tokens` / `promptTokens` / `input_tokens` / `inputTokens` nas observations do Langfuse; +2. `usage.input` ou `usageDetails.input`; +3. `metadata.input_size` emitido pelo framework; +4. fallback para tamanho textual de `input_text`, `output_text` e `messages`. + +Exemplo: + +```text +promptLength = 732 +``` + +#### loop + +O campo `loop` usa o detector VLoop. + +```text +0 = sem loop detectado +1 = loop detectado +``` + +--- + +### 5.12 VLoop Analytics + +Arquivo principal: + +```text +evaluator/analytics/vloop.py +``` + +Responsável por detectar repetição/loop conversacional em padrão semelhante ao guardrail VLoop do `agent_framework_oci`. + +A função normalmente exposta é: + +```python +vloop_flag(raw) -> int +``` + +Ela retorna: + +```text +0 quando não há evidência de loop +1 quando há repetição suspeita +``` + +--- + +### 5.13 Langfuse Score Publisher + +Arquivo principal: + +```text +evaluator/publishers/langfuse_scores.py +``` + +Responsável por publicar scores de avaliação de volta no Langfuse, quando habilitado. + +Variável de controle: + +```env +PUBLISH_LANGFUSE_SCORES=true +``` + +Quando desabilitado, o evaluator apenas grava no banco e exporta arquivo. + +--- + +## 6. Estrutura de diretórios + +```text +agent_framework_evaluator/ +├── configs/ +│ ├── identity.yaml +│ └── judge/ +│ └── agents.yaml +├── docs/ +├── evaluator/ +│ ├── __init__.py +│ ├── cli.py +│ ├── engine.py +│ ├── api/ +│ │ └── main.py +│ ├── analytics/ +│ │ └── vloop.py +│ ├── collectors/ +│ │ ├── base.py +│ │ ├── langfuse.py +│ │ ├── agent_framework.py +│ │ └── mock.py +│ ├── config/ +│ │ ├── settings.py +│ │ └── agents.py +│ ├── core/ +│ │ └── models.py +│ ├── identity/ +│ │ └── resolver.py +│ ├── judges/ +│ │ └── llm_judge.py +│ ├── llm/ +│ │ ├── client.py +│ │ └── oci_openai.py +│ ├── output/ +│ │ └── legacy_exporter.py +│ ├── persistence/ +│ │ ├── oracle_store.py +│ │ └── repository.py +│ ├── prompts/ +│ │ ├── loader.py +│ │ ├── trace_judge_prompt.md +│ │ └── session_judge_prompt.md +│ └── publishers/ +│ └── langfuse_scores.py +├── output/ +├── Dockerfile +├── docker-compose.yml +├── pyproject.toml +└── README.md +``` + +--- + +## 7. Configuração + +### 7.1 Arquivo `.env` + +Exemplo: + +```env +# Oracle / ADB +ADB_USER=ADMIN +ADB_PASSWORD=your_password +ADB_DSN=oradb23ai_high +ADB_WALLET_DIR=/path/to/Wallet_ORADB23ai +DB_TABLE_PREFIX=AGENTFW_ + +# Langfuse +LANGFUSE_ENABLED=true +LANGFUSE_HOST=http://localhost:3005 +LANGFUSE_PUBLIC_KEY=pk-lf-... +LANGFUSE_SECRET_KEY=sk-lf-... +PUBLISH_LANGFUSE_SCORES=false + +# LLM +LLM_PROVIDER=oci_openai +OCI_GENAI_ENDPOINT=https://... +OCI_GENAI_MODEL_ID=... +OCI_GENAI_API_KEY=... +OCI_GENAI_COMPARTMENT_ID=... + +# Evaluator +EVALUATOR_OUTPUT_DIR=output +EVALUATOR_BATCH_SIZE=10 +EVALUATOR_MAX_ATTEMPTS=2 +EVALUATOR_AGENTS_CONFIG=configs/judge/agents.yaml +IDENTITY_CONFIG_PATH=configs/identity.yaml +TRACE_PROMPT_PATH=evaluator/prompts/trace_judge_prompt.md +SESSION_PROMPT_PATH=evaluator/prompts/session_judge_prompt.md +``` + +--- + +### 7.2 Configuração de agentes + +Arquivo: + +```text +configs/judge/agents.yaml +``` + +Exemplo: + +```yaml +agents: + - agent_id: telecom_contas + enabled: true + aliases: + - telecom_contas + - billing_agent + - financeiro_agent + percentage: 1.0 + + - agent_id: retail_orders + enabled: true + aliases: + - retail_orders + - orders_agent + percentage: 1.0 + + - agent_id: financeiro_agent + enabled: true + aliases: + - financeiro_agent + percentage: 1.0 +``` + +O campo `aliases` é importante porque o Langfuse pode registrar o agente de formas diferentes, por exemplo: + +```text +agent_id = telecom_contas +route = financeiro_agent +agent = financeiro_agent +``` + +--- + +### 7.3 Configuração de identidade + +Arquivo: + +```text +configs/identity.yaml +``` + +O evaluator deve usar o mesmo padrão do framework. + +Exemplo: + +```yaml +identity: + version: 2 + keys: + customer_key: + sources: + - business_context.customer_key + - metadata.customer_key + - user_id + + contract_key: + sources: + - business_context.contract_key + - metadata.contract_key + + interaction_key: + sources: + - business_context.interaction_key + - metadata.ura_call_id + - metadata.message_id + - message_id + + session_key: + sources: + - business_context.session_key + - metadata.session_key + - session_id + - conversation_key +``` + +O campo `interaction_key` é usado para preencher o `uraCallId` no export legado. + +--- + +## 8. Como executar + +### 8.1 Instalar dependências + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +Se estiver usando Conda: + +```bash +conda activate py313 +pip install -e . +``` + +--- + +### 8.2 Validar configuração + +```bash +python -m evaluator.cli show-config +``` + +Saída esperada: + +```text +{ + "env_path": ".../.env", + "adb_dsn": "oradb23ai_high", + "wallet": ".../Wallet_ORADB23ai", + "langfuse": true, + "publish_langfuse_scores": false, + "llm_provider": "oci_openai", + "agents_config": "configs/judge/agents.yaml" +} +``` + +--- + +### 8.3 Criar/validar schema + +```bash +python -m evaluator.cli init-db +``` + +Saída esperada: + +```text +{'status': 'OK', 'message': 'Evaluator schema checked/created successfully.'} +``` + +--- + +### 8.4 Rodar avaliação por período + +```bash +python -m evaluator.cli run \ + --period-start 2026-06-11T00:00:00 \ + --period-end 2026-06-12T00:00:00 \ + --source langfuse +``` + +--- + +### 8.5 Rodar avaliação por agentes configurados + +```bash +python -m evaluator.cli run-agents --source langfuse +``` + +Saída esperada: + +```text +[ + { + 'status': 'COMPLETED', + 'run_id': '...', + 'total_items': 19, + 'completed_items': 19, + 'failed_items': 0, + 'evaluations': 19, + 'avg_score': 0.72, + 'agent_id': 'telecom_contas', + 'output_file': 'output/AGENTE_telecom_contas_LLM_JUDGE_20260612.TXT.GZ', + 'uploaded_to': None + } +] +``` + +--- + +### 8.6 Consultar progresso + +```bash +python -m evaluator.cli progress +``` + +Ou via API: + +```bash +curl http://localhost:8001/runs//progress +``` + +--- + +### 8.7 Ver arquivo exportado + +```bash +gzip -cd output/AGENTE_telecom_contas_LLM_JUDGE_20260612.TXT.GZ +``` + +Exemplo de linha válida: + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"TOTAL"|;"19" +``` + +--- + +## 9. Banco de dados + +### 9.1 Tabelas principais + +#### EVALUATION_RUN + +Armazena uma execução de avaliação. + +Campos principais: + +```text +RUN_ID +PERIOD_START +PERIOD_END +SOURCE +AGENT_ID +STATUS +TOTAL_ITEMS +PROCESSED_ITEMS +FAILED_ITEMS +LAST_HEARTBEAT_AT +CREATED_AT +UPDATED_AT +ERROR_MESSAGE +``` + +--- + +#### EVALUATION_ITEM + +Armazena cada conversa/turno coletado. + +Campos principais: + +```text +ITEM_ID +RUN_ID +TRACE_ID +SESSION_ID +MESSAGE_ID +AGENT_ID +CHANNEL +STATUS +ATTEMPT_COUNT +RAW_JSON +CREATED_AT +UPDATED_AT +ERROR_MESSAGE +``` + +--- + +#### EVALUATION_RESULT + +Armazena resultados de trace e sessão. + +Campos principais: + +```text +RESULT_ID +RUN_ID +ITEM_ID +TRACE_ID +SESSION_ID +AGENT_ID +JUDGE_TYPE +JUDGE_NAME +JUDGE_SCORE +ACCURACY_SCORE +ALUCINATION_SCORE +INFERRED_CSI_SCORE +RESOLUTION +CONVERSATION_PRECISION +RATIONALE +RESULT_JSON +CREATED_AT +``` + +`JUDGE_TYPE` pode ser: + +```text +TRACE +SESSION +``` + +--- + +#### EVALUATION_PROGRESS_EVENT + +Armazena eventos de progresso da execução. + +Exemplos de stage: + +```text +RUN_CREATED +COLLECTING +COLLECTED +SAMPLED +ITEMS_INSERTED +BATCH_STARTED +ITEM_COMPLETED +ITEM_FAILED +SESSION_JUDGE_COMPLETED +EXPORTED +COMPLETED +PARTIAL +``` + +--- + +## 10. Como os códigos funcionam em conjunto + +### 10.1 Fluxo completo de execução + +```text +CLI run-agents + ↓ +carrega configs/judge/agents.yaml + ↓ +para cada agente habilitado + ↓ +EvaluationEngine.run_agent(agent) + ↓ +cria EVALUATION_RUN + ↓ +LangfuseCollector.collect(...) + ↓ +IdentityResolver.resolve(...) + ↓ +ConversationRecord + ↓ +EvaluationRepository.insert_items(...) + ↓ +EvaluationEngine._process(run_id) + ↓ +TIMStyleLLMJudge.judge_trace(record) + ↓ +LLMClient.complete(prompt) + ↓ +save_trace_result(...) + ↓ +TIMStyleLLMJudge.judge_sessions(records) + ↓ +save_session_result(...) + ↓ +export_legacy_txt_gz(...) + ↓ +COMPLETED +``` + +--- + +### 10.2 Papel do collector + +O collector é responsável por transformar dados externos em dados canônicos. + +Ele deve esconder diferenças entre fontes como: + +```text +Langfuse +agent_framework database +mock data +``` + +A saída sempre deve ser: + +```python +ConversationRecord +``` + +--- + +### 10.3 Papel do judge + +O judge recebe um `ConversationRecord`, monta um prompt e chama o LLM. + +Ele não deve conhecer Oracle, Langfuse, export legado ou API. + +Ele só avalia. + +--- + +### 10.4 Papel do repository + +O repository é a camada de persistência. + +Ele não deve conter regra de negócio de avaliação. + +Ele apenas grava, busca e atualiza dados. + +--- + +### 10.5 Papel do exporter + +O exporter transforma dados persistidos em arquivo legado. + +Ele não deve resolver identidade de forma complexa. + +O ideal é que ele leia campos já normalizados: + +```text +metadata.business_context.interaction_key +metadata.channelId +metadata.messageId +metadata.promptLength +``` + +No entanto, para resiliência, ele também pode consultar `RAW_JSON` como fallback. + +--- + +## 11. Regras importantes de desenho + +### 11.1 O evaluator não deve ficar chumbado para um agente + +Evite lógica como: + +```python +if agent_id == "telecom_contas": + ura_call_id = metadata["ura_call_id"] +``` + +O correto é usar `identity.yaml`. + +--- + +### 11.2 O exporter não deve conhecer detalhes internos do Langfuse + +Evite acoplamento excessivo a caminhos como: + +```text +raw.detail.observations[0].metadata.ura_call_id +raw.trace.input.business_context.interaction_key +``` + +Isso deve ser resolvido no collector. + +--- + +### 11.3 `promptLength` deve vir de tokens quando possível + +Prioridade recomendada: + +```text +1. prompt_tokens / promptTokens +2. input_tokens / inputTokens +3. usage.input / usageDetails.input +4. metadata.input_size +5. tamanho textual de input/output/messages +``` + +--- + +### 11.4 `uraCallId` deve vir do BusinessContext + +O campo legado `uraCallId` deve ser mapeado para: + +```text +business_context.interaction_key +``` + +Esse é o nome canônico do framework. + +--- + +### 11.5 `sessionId` deve vir do BusinessContext + +O campo legado `sessionId` deve ser mapeado para: + +```text +business_context.session_key +``` + +Não confundir com a chave composta completa: + +```text +default:telecom_contas: +``` + +O evaluator pode guardar a chave completa, mas o export legado normalmente deve usar o identificador de sessão limpo. + +--- + +## 12. Testes recomendados + +### 12.1 Teste de configuração + +```bash +python -m evaluator.cli show-config +``` + +Validar: + +```text +ADB_DSN +Wallet +Langfuse enabled +LLM provider +Agents config +Identity config +``` + +--- + +### 12.2 Teste de banco + +```bash +python -m evaluator.cli init-db +``` + +Depois validar tabelas: + +```sql +select table_name +from user_tables +where table_name like 'AGENTFW_EVALUATION%'; +``` + +--- + +### 12.3 Teste com mock + +```bash +python -m evaluator.cli run --source mock +``` + +Use esse teste para validar schema, judge e export sem depender do Langfuse. + +--- + +### 12.4 Teste com Langfuse + +```bash +python -m evaluator.cli run-agents --source langfuse +``` + +Validar: + +```text +total_items > 0 +completed_items > 0 +failed_items = 0 +evaluations > 0 +output_file preenchido +``` + +--- + +### 12.5 Teste do export + +```bash +gzip -cd output/AGENTE_telecom_contas_LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +Validar colunas: + +```text +judgeScore preenchido +accuracyScore preenchido +alucinationScore preenchido +promptLength maior que 0 +loop 0 ou 1 +inferredCsiScore preenchido +resolution 0 ou 1 +conversationPrecision 0 ou 1 +uraCallId preenchido +channelId preenchido +sessionId preenchido +messageId preenchido +``` + +--- + +## 13. Troubleshooting + +### 13.1 `promptLength` sai 0 + +Causas comuns: + +- `find_prompt_tokens` não foi incluído no arquivo; +- `promptTokens` está zerado no Langfuse; +- `input_size` não está sendo percorrido; +- `RAW_JSON` está vindo como string não convertida; +- exporter antigo ainda está rodando; +- `except Exception: pass` está mascarando erro. + +Debug recomendado: + +```python +print("PROMPT_LENGTH", extract_prompt_length(raw)) +print("RAW_TYPE", type(raw)) +print("RAW_KEYS", list(raw.keys())[:20]) +``` + +--- + +### 13.2 `uraCallId` sai vazio + +Causas comuns: + +- `identity.yaml` não está sendo carregado; +- collector não está copiando `business_context` para `metadata`; +- `interaction_key` não existe no trace; +- exporter não usa `business_context.interaction_key`. + +Validação: + +```sql +select RAW_JSON +from AGENTFW_EVALUATION_ITEM +where MESSAGE_ID = ''; +``` + +Procurar: + +```text +interaction_key +ura_call_id +business_context +``` + +--- + +### 13.3 `ORA-00904 invalid identifier` + +Geralmente indica schema antigo. + +Exemplos já encontrados: + +```text +ORA-00904: UPDATED_AT invalid identifier +ORA-00904: REASONING invalid identifier +ORA-00904: JUDGE_TYPE invalid identifier +``` + +Correção: + +```bash +python -m evaluator.cli init-db +``` + +Se a tabela já existir sem a coluna nova, o `_init_schema` precisa executar `ALTER TABLE ADD` de forma idempotente. + +--- + +### 13.4 `ORA-00054 resource busy` + +Indica lock em tabela. + +Causas comuns: + +- API rodando enquanto `init-db` tenta alterar schema; +- outro processo usando a tabela; +- transação aberta no SQL Developer. + +Correção: + +1. parar API/CLI; +2. fechar sessões abertas; +3. executar novamente `init-db`. + +--- + +### 13.5 `OCI LLM 401` + +Indica problema de autenticação no LLM. + +Validar: + +```env +OCI_GENAI_ENDPOINT +OCI_GENAI_MODEL_ID +OCI_GENAI_API_KEY +OCI_GENAI_COMPARTMENT_ID +``` + +Também confirmar se o evaluator está lendo o `.env` correto: + +```bash +python -m evaluator.cli show-config +``` + +--- + +### 13.6 `Entity with key ${OCI_GENAI_MODEL_ID} not found` + +Indica que o valor literal `${OCI_GENAI_MODEL_ID}` chegou ao provider. + +Causas comuns: + +- variável não expandida; +- YAML usando `${OCI_GENAI_MODEL_ID}` sem interpolação; +- `.env` não carregado; +- configuração do LLM client não resolve placeholders. + +Correção: + +- colocar o model ID real no `.env`; +- garantir interpolação em `settings.py`; +- validar com `show-config`. + +--- + +## 14. Checklist de validação final + +Antes de considerar o evaluator pronto, validar: + +```text +[ ] init-db executa sem erro +[ ] show-config mostra .env correto +[ ] Langfuse retorna traces +[ ] run-agents coleta itens por agente +[ ] LLM Judge responde JSON válido +[ ] EVALUATION_RESULT grava TRACE e SESSION +[ ] progress mostra eventos úteis +[ ] export TXT.GZ é gerado +[ ] promptLength > 0 +[ ] uraCallId preenchido +[ ] sessionId preenchido +[ ] messageId preenchido +[ ] loop preenchido com 0 ou 1 +[ ] arquivo termina com TOTAL +[ ] scores podem ser publicados no Langfuse quando habilitado +``` + +--- + +## 15. Exemplo de resultado final validado + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"0.9"|;"1"|;"0"|;"642"|;"0"|;"0.5"|;"1"|;"1"|;"5ab3ea80-7428-402f-98ec-04e7cd5327e4"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"5ab3ea80-7428-402f-98ec-04e7cd5327e4" +"TOTAL"|;"19" +``` + +Esse resultado indica: + +- Judge funcionando; +- prompt tokens extraídos corretamente; +- VLoop sem ocorrência; +- métricas de sessão preenchidas; +- identidade canônica funcionando; +- export legado no layout esperado. + +--- + +## 16. Resumo executivo + +O `agent_framework_evaluator` é a camada batch/offline de avaliação do ecossistema `agent_framework_oci`. + +Ele consome a telemetria gerada pelo framework, aplica um Judge LLM com regras de avaliação, persiste resultados em Oracle/ADB, gera arquivo e pode republicar scores no Langfuse. + +A arquitetura correta separa responsabilidades: + +```text +Collector normaliza dados +IdentityResolver resolve identidade +Judge avalia conversa +Repository persiste +Exporter gera legado +API/CLI operam a solução +``` + +Com isso, o evaluator fica genérico para múltiplos agentes e evita acoplamento direto a formatos específicos de trace ou payload. diff --git a/configs/.DS_Store b/configs/.DS_Store new file mode 100644 index 0000000..80b3397 Binary files /dev/null and b/configs/.DS_Store differ diff --git a/configs/identity.yaml b/configs/identity.yaml new file mode 100644 index 0000000..5f20147 --- /dev/null +++ b/configs/identity.yaml @@ -0,0 +1,55 @@ +identity: + version: "2" + required: + - session_key + keys: + customer_key: + description: Cliente/assinante/consumidor canônico. + sources: + - business_context.customer_key + - customer_key + - msisdn + - customer_id + - user_id + - ani + - from + contract_key: + description: Contrato, conta, fatura, pedido ou asset principal. + sources: + - business_context.contract_key + - contract_key + - invoice_id + - current_invoice_number + - order_id + - pedido_id + - asset_id + interaction_key: + description: Chave externa da interação/call/chat vinda do canal. + sources: + - business_context.interaction_key + - interaction_key + - ura_call_id + - call_id + - message_id + account_key: + description: Conta de cobrança/conta comercial. + sources: + - business_context.account_key + - account_key + - account_id + - billing_account_id + resource_key: + description: Recurso/linha/produto/asset específico. + sources: + - business_context.resource_key + - resource_key + - asset_id + - product_id + - sku + session_key: + description: Sessão técnica estável já escopada por tenant e agente. + sources: + - business_context.session_key + - session_key + - conversation_key + - session_id diff --git a/configs/judge/agents.yaml b/configs/judge/agents.yaml new file mode 100644 index 0000000..df8bc1a --- /dev/null +++ b/configs/judge/agents.yaml @@ -0,0 +1,21 @@ +agents: + - agent_id: telecom_contas + enabled: true + days_back: 1 + percentage: 1.0 + langfuse_agent_aliases: [telecom_contas, billing_agent, financeiro_agent] + gcs_prefix: agnt_ai_contas/llm_qa/input + + - agent_id: retail_orders + enabled: true + days_back: 1 + percentage: 1.0 + langfuse_agent_aliases: [retail_orders, orders_agent, retail_agent] + gcs_prefix: agnt_ai_orders/llm_qa/input + + - agent_id: financeiro_agent + enabled: true + days_back: 1 + percentage: 1.0 + langfuse_agent_aliases: [financeiro_agent, billing_agent, telecom_contas] + gcs_prefix: agnt_ai_financeiro/llm_qa/input diff --git a/configs/judge/session_metrics.yaml b/configs/judge/session_metrics.yaml new file mode 100644 index 0000000..44a4831 --- /dev/null +++ b/configs/judge/session_metrics.yaml @@ -0,0 +1,17 @@ +session_metrics: | + Você é um avaliador imparcial de uma sessão completa de conversa entre + cliente e agente. Vai receber a TRANSCRIÇÃO da sessão (alternância + user/agent). Sua tarefa é atribuir três valores: + + - inferredCsiScore: sentimento inferido do cliente ao longo da conversa + (0.0 negativo, 0.5 neutro, 1.0 positivo). + - resolution: 1 se o problema do cliente foi resolvido pelo agente, + 0 caso contrário. + - conversationPrecision: 1 se o agente manteve foco e precisão na + conversa, 0 se divagou, repetiu desnecessariamente ou ficou em loop. + + Retorne SOMENTE um JSON válido, sem texto adicional: + + {"inferredCsiScore": , "resolution": <0|1>, "conversationPrecision": <0|1>, "rationale": ""} + + rationale deve ter no máximo 200 caracteres. diff --git a/configs/judge/trace_metrics.yaml b/configs/judge/trace_metrics.yaml new file mode 100644 index 0000000..6e73b5a --- /dev/null +++ b/configs/judge/trace_metrics.yaml @@ -0,0 +1,16 @@ +trace_metrics: | + Você é um avaliador imparcial de respostas de agentes conversacionais. + Vai receber o HISTÓRICO da conversa (últimos turnos), a MENSAGEM DO + USUÁRIO e a RESPOSTA DO AGENTE. Sua tarefa é atribuir três notas + numéricas entre 0.0 e 1.0: + + - judgeScore: qualidade global da resposta (clareza, utilidade, tom). + - accuracyScore: factualidade da resposta frente ao contexto fornecido. + - alucinationScore: grau em que a resposta contém afirmações NÃO + suportadas pelo contexto (alto = mais alucinação = pior). + + Retorne SOMENTE um JSON válido, sem nenhum texto adicional, no formato: + + {"judgeScore": , "accuracyScore": , "alucinationScore": , "rationale": ""} + + rationale deve ter no máximo 200 caracteres. diff --git a/configs/llm_profiles/llm_profiles.yaml b/configs/llm_profiles/llm_profiles.yaml new file mode 100644 index 0000000..7975152 --- /dev/null +++ b/configs/llm_profiles/llm_profiles.yaml @@ -0,0 +1,11 @@ +profiles: + judge: + provider: oci_openai + model: meta.llama-3.3-70b-instruct + temperature: 0 + max_tokens: 900 + mock: + provider: mock + model: mock-judge + temperature: 0 + max_tokens: 300 diff --git a/evaluator/.DS_Store b/evaluator/.DS_Store new file mode 100644 index 0000000..8b2d75b Binary files /dev/null and b/evaluator/.DS_Store differ diff --git a/evaluator/__init__.py b/evaluator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evaluator/__pycache__/__init__.cpython-313.pyc b/evaluator/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..fd2e756 Binary files /dev/null and b/evaluator/__pycache__/__init__.cpython-313.pyc differ diff --git a/evaluator/__pycache__/cli.cpython-313.pyc b/evaluator/__pycache__/cli.cpython-313.pyc new file mode 100644 index 0000000..35d2fa7 Binary files /dev/null and b/evaluator/__pycache__/cli.cpython-313.pyc differ diff --git a/evaluator/__pycache__/engine.cpython-313.pyc b/evaluator/__pycache__/engine.cpython-313.pyc new file mode 100644 index 0000000..17434e3 Binary files /dev/null and b/evaluator/__pycache__/engine.cpython-313.pyc differ diff --git a/evaluator/analytics/__init__.py b/evaluator/analytics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evaluator/analytics/__pycache__/__init__.cpython-313.pyc b/evaluator/analytics/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..b082518 Binary files /dev/null and b/evaluator/analytics/__pycache__/__init__.cpython-313.pyc differ diff --git a/evaluator/analytics/__pycache__/vloop.cpython-313.pyc b/evaluator/analytics/__pycache__/vloop.cpython-313.pyc new file mode 100644 index 0000000..13b3e01 Binary files /dev/null and b/evaluator/analytics/__pycache__/vloop.cpython-313.pyc differ diff --git a/evaluator/analytics/vloop.py b/evaluator/analytics/vloop.py new file mode 100644 index 0000000..3964f8a --- /dev/null +++ b/evaluator/analytics/vloop.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import re +from typing import Any + + +def _normalize(text: Any) -> str: + """Same deterministic spirit as Agent Framework VLOOP: lower + strip. + + We also collapse whitespace because offline telemetry can contain line breaks, + repeated spaces, or formatting differences from Langfuse observations. + """ + if text is None: + return "" + return re.sub(r"\s+", " ", str(text).lower()).strip() + + +def _message_role(message: Any) -> str: + if isinstance(message, dict): + return str(message.get("role") or "").lower() + return str(getattr(message, "role", "") or "").lower() + + +def _message_content(message: Any) -> str: + if isinstance(message, dict): + return str(message.get("content") or "") + return str(getattr(message, "content", "") or "") + + +def user_texts_from_record(record_or_raw: Any) -> list[str]: + """Extract user/human texts from ConversationRecord or its JSON dict.""" + if isinstance(record_or_raw, dict): + messages = record_or_raw.get("messages") or [] + input_text = record_or_raw.get("input_text") or "" + else: + messages = getattr(record_or_raw, "messages", []) or [] + input_text = getattr(record_or_raw, "input_text", "") or "" + + out: list[str] = [] + for message in messages: + role = _message_role(message) + if role in {"user", "human", "cliente", "customer"}: + text = _normalize(_message_content(message)) + if text: + out.append(text) + + # Ensure the canonical current user input participates even if messages were + # reconstructed only from observations and missed the trace-level input. + canonical = _normalize(input_text) + if canonical and canonical not in out: + out.append(canonical) + return out + + +def detect_vloop(record_or_raw: Any, history_window: int = 6, min_previous_repetitions: int = 2) -> bool: + """Offline equivalent of Agent Framework VLOOP. + + Framework logic: + normalized = lower(text).strip() + history = lower(history_texts)[-6:] + repeated = history.count(normalized) >= 2 + + Offline telemetry does not provide the exact guardrail context, so we rebuild + it from user messages. The current user text is the last user message. The + previous history is the prior messages in the same reconstructed trace/session. + """ + texts = user_texts_from_record(record_or_raw) + if not texts: + return False + + current = texts[-1] + if not current: + return False + + history = texts[:-1][-history_window:] + if history.count(current) >= min_previous_repetitions: + return True + + # Defensive fallback: if the reconstructed messages do not preserve a clear + # current turn, flag any user utterance repeated 3+ times in the recent window. + recent = texts[-(history_window + 1):] + return any(recent.count(t) >= (min_previous_repetitions + 1) for t in set(recent) if t) + + +def vloop_flag(record_or_raw: Any) -> int: + return 1 if detect_vloop(record_or_raw) else 0 diff --git a/evaluator/api/__pycache__/main.cpython-313.pyc b/evaluator/api/__pycache__/main.cpython-313.pyc new file mode 100644 index 0000000..448258a Binary files /dev/null and b/evaluator/api/__pycache__/main.cpython-313.pyc differ diff --git a/evaluator/api/main.py b/evaluator/api/main.py new file mode 100644 index 0000000..b20b028 --- /dev/null +++ b/evaluator/api/main.py @@ -0,0 +1,37 @@ +from __future__ import annotations +from fastapi import FastAPI +from fastapi.responses import HTMLResponse +from evaluator.persistence.repository import EvaluationRepository +from fastapi import Request +from fastapi.responses import JSONResponse +import traceback + +app = FastAPI(title='Agent Framework Evaluator') + +@app.get('/health') +def health(): return {'status':'ok'} + +@app.get('/runs') +async def runs(limit:int=20): return await EvaluationRepository(auto_init_schema=False).alist_runs(limit) + +@app.get('/runs/{run_id}/progress') +async def run_progress(run_id:str, events:int=20): + return await EvaluationRepository(auto_init_schema=False).aget_run_progress(run_id, events) + +@app.get("/runs/{run_id}/results") +async def results(run_id: str, limit: int = 100): + return await EvaluationRepository(auto_init_schema=False).alist_results(run_id, limit) + +@app.get('/ui', response_class=HTMLResponse) +def ui(): + return '''Agent Framework Evaluator

Agent Framework Evaluator

Offline LLM-as-a-Judge with Agent Framework telemetry.

Run	Agent	Source	Status	Total	Processed	Failed	Created

''' + +@app.exception_handler(Exception) +async def debug_exception_handler(request: Request, exc: Exception): + return JSONResponse( + status_code=500, + content={ + "error": str(exc), + "traceback": traceback.format_exc(), + }, + ) diff --git a/evaluator/cli.py b/evaluator/cli.py new file mode 100644 index 0000000..22a017f --- /dev/null +++ b/evaluator/cli.py @@ -0,0 +1,80 @@ +from __future__ import annotations +import asyncio +from datetime import datetime, timedelta +import typer +from rich import print +from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeElapsedColumn +from evaluator.config.agents import load_agents +from evaluator.engine import EvaluationEngine +from evaluator.persistence.repository import EvaluationRepository +from evaluator.config.settings import settings + +app = typer.Typer(help='Agent Framework TIM-style LLM Judge Evaluator') + + +def _run_progress(coro_factory): + async def runner(): + state={'run_id': None} + with Progress(SpinnerColumn(), TextColumn('[bold blue]{task.fields[stage]}'), BarColumn(), TextColumn('{task.completed}/{task.total}'), TextColumn('{task.percentage:>3.0f}%'), TimeElapsedColumn()) as progress: + task=progress.add_task('evaluation', total=1, stage='starting') + async def cb(event): + state['run_id'] = event.get('run_id') or state['run_id'] + stage = event.get('stage','') + msg = event.get('message','') + if state['run_id']: + snap = await EvaluationRepository(auto_init_schema=False).aget_run_progress(state['run_id'], event_limit=1) + total=int(snap.get('total_items') or 0) or 1 + done=int(snap.get('done_items') or 0) + progress.update(task,total=total,completed=done,stage=f'{stage}: {msg}'[:120]) + result = await coro_factory(cb) + progress.update(task, completed=1, total=1, stage='finished') + return result + return asyncio.run(runner()) + +@app.command("reset-db") +def reset_db(): + repo = EvaluationRepository(auto_init_schema=False) + repo.store.drop_schema() + repo.store._init_schema() + print({"status": "OK", "message": "Evaluator schema dropped and recreated successfully."}) + +@app.command('init-db') +def init_db(): + EvaluationRepository(auto_init_schema=True) + print({'status':'OK','message':'schema checked/created'}) + +@app.command('show-config') +def show_config(): + print({'env_path': str(settings.project_root / '.env'), 'adb_dsn': settings.ADB_DSN, 'wallet': settings.ADB_WALLET_LOCATION, 'langfuse': settings.enable_langfuse, 'publish_langfuse_scores': settings.publish_langfuse_scores, 'llm_provider': settings.llm_provider, 'llm_profile': settings.llm_profile, 'oci_genai_base_url': settings.OCI_GENAI_BASE_URL, 'oci_genai_model': settings.OCI_GENAI_MODEL, 'oci_genai_api_key_configured': bool(settings.OCI_GENAI_API_KEY), 'agents_config': settings.agents_config_path}) + +@app.command('run') +def run(period_start: datetime, period_end: datetime, source: str='langfuse', limit: int|None=None, show_progress: bool=True): + if show_progress: + result = _run_progress(lambda cb: EvaluationEngine(progress_callback=cb).run(period_start, period_end, source, limit)) + else: + result = asyncio.run(EvaluationEngine().run(period_start, period_end, source, limit)) + print(result) + +@app.command('run-agents') +def run_agents(source: str='langfuse', agent_id: str|None=None, limit: int|None=None): + async def main(): + results=[] + now=datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + for agent in load_agents(): + if agent_id and agent.agent_id != agent_id: continue + start = now - timedelta(days=agent.days_back) + engine=EvaluationEngine() + results.append(await engine.run_agent(agent, start, now, source=source, limit=limit)) + return results + print(asyncio.run(main())) + +@app.command('progress') +def progress(run_id: str, events: int=20): + print(asyncio.run(EvaluationRepository(auto_init_schema=False).aget_run_progress(run_id, event_limit=events))) + +@app.command('runs') +def runs(limit: int=20): + print(asyncio.run(EvaluationRepository(auto_init_schema=False).alist_runs(limit))) + +if __name__ == '__main__': + app() diff --git a/evaluator/collectors/.DS_Store b/evaluator/collectors/.DS_Store new file mode 100644 index 0000000..eaa3227 Binary files /dev/null and b/evaluator/collectors/.DS_Store differ diff --git a/evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc b/evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc new file mode 100644 index 0000000..3548d6d Binary files /dev/null and b/evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc differ diff --git a/evaluator/collectors/__pycache__/base.cpython-313.pyc b/evaluator/collectors/__pycache__/base.cpython-313.pyc new file mode 100644 index 0000000..423b8fd Binary files /dev/null and b/evaluator/collectors/__pycache__/base.cpython-313.pyc differ diff --git a/evaluator/collectors/__pycache__/langfuse.cpython-313.pyc b/evaluator/collectors/__pycache__/langfuse.cpython-313.pyc new file mode 100644 index 0000000..cbbe06f Binary files /dev/null and b/evaluator/collectors/__pycache__/langfuse.cpython-313.pyc differ diff --git a/evaluator/collectors/__pycache__/mock.cpython-313.pyc b/evaluator/collectors/__pycache__/mock.cpython-313.pyc new file mode 100644 index 0000000..5dd4519 Binary files /dev/null and b/evaluator/collectors/__pycache__/mock.cpython-313.pyc differ diff --git a/evaluator/collectors/agent_framework.py b/evaluator/collectors/agent_framework.py new file mode 100644 index 0000000..1b5602a --- /dev/null +++ b/evaluator/collectors/agent_framework.py @@ -0,0 +1,42 @@ +from __future__ import annotations +from datetime import datetime +from evaluator.collectors.base import ConversationCollector +from evaluator.core.models import ConversationRecord, ConversationMessage +from evaluator.persistence.oracle_store import OracleStore, _json_loads +from evaluator.config.settings import settings + +class AgentFrameworkCollector(ConversationCollector): + def __init__(self): + self.store = OracleStore(settings, auto_init_schema=False) + + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None): + return await self.store.to_thread(self._collect, period_start, period_end, agent_aliases or set(), limit or 100) + + def _collect(self, period_start, period_end, aliases, limit): + records=[] + with self.store.connect() as conn: + cur=conn.cursor() + cur.execute(f""" + select * from ( + select SESSION_ID, AGENT_ID, CHANNEL, CONTEXT_JSON, METADATA_JSON, CREATED_AT + from {self.store.t('AGENT_SESSION')} + where CREATED_AT >= :start_at and CREATED_AT < :end_at + order by CREATED_AT desc + ) where rownum <= :max_rows + """, dict(start_at=period_start, end_at=period_end, max_rows=limit)) + sessions=cur.fetchall() + for session_id, agent_id, channel, ctx, meta, created_at in sessions: + if aliases and agent_id not in aliases: continue + cur.execute(f""" + select ROLE, CONTENT, METADATA_JSON, CREATED_AT, MESSAGE_ID + from {self.store.t('AGENT_MESSAGE')} + where SESSION_ID=:session_id order by CREATED_AT + """, dict(session_id=session_id)) + rows=cur.fetchall() + msgs=[] + for role, content, msg_meta, msg_created, message_id in rows: + msgs.append(ConversationMessage(role=role, content=content or '', created_at=str(msg_created), metadata=_json_loads(msg_meta.read() if hasattr(msg_meta,'read') else msg_meta,{}))) + input_text=next((m.content for m in msgs if m.role in ('user','human')), '') + output_text=next((m.content for m in reversed(msgs) if m.role in ('assistant','ai','agent')), '') + records.append(ConversationRecord(session_id=session_id, trace_id=session_id, message_id=rows[-1][4] if rows else None, agent_id=agent_id, channel=channel, input_text=input_text, output_text=output_text, messages=msgs, metadata=_json_loads(meta.read() if hasattr(meta,'read') else meta,{}), raw={})) + return records diff --git a/evaluator/collectors/base.py b/evaluator/collectors/base.py new file mode 100644 index 0000000..cc00c1f --- /dev/null +++ b/evaluator/collectors/base.py @@ -0,0 +1,8 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from datetime import datetime +from evaluator.core.models import ConversationRecord + +class ConversationCollector(ABC): + @abstractmethod + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None) -> list[ConversationRecord]: ... diff --git a/evaluator/collectors/langfuse.py b/evaluator/collectors/langfuse.py new file mode 100644 index 0000000..95fabfc --- /dev/null +++ b/evaluator/collectors/langfuse.py @@ -0,0 +1,355 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +import httpx + +from evaluator.collectors.base import ConversationCollector +from evaluator.config.settings import settings +from evaluator.core.models import ConversationMessage, ConversationRecord +from evaluator.identity.resolver import IdentityResolver +from evaluator.config.settings import settings + +def _iso_z(dt: datetime) -> str: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _metadata(obj: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(obj, dict): + return {} + meta = obj.get("metadata") or {} + return meta if isinstance(meta, dict) else {} + + +def _first_value(*values: Any) -> Any: + for value in values: + if value not in (None, "", [], {}): + return value + return None + + +def _content_to_text(value: Any) -> str: + """Convert Langfuse/OpenAI-style content payloads to plain text.""" + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, (int, float, bool)): + return str(value) + if isinstance(value, list): + parts: list[str] = [] + for item in value: + text = _content_to_text(item) + if text: + parts.append(text) + return "\n".join(parts) + if isinstance(value, dict): + # OpenAI multimodal content often comes as {"type":"text","text":"..."} + for key in ("text", "content", "message", "value", "input", "output", "completion"): + if key in value: + text = _content_to_text(value.get(key)) + if text: + return text + # Chat completion response variants. + choices = value.get("choices") + if isinstance(choices, list) and choices: + return _content_to_text(choices[0]) + msg = value.get("message") + if isinstance(msg, dict): + return _content_to_text(msg.get("content")) + return "" + return str(value) + + +def _messages_from_value(value: Any, default_role: str) -> list[ConversationMessage]: + """Extract chat messages from strings/lists/dicts returned by Langfuse.""" + if value in (None, "", [], {}): + return [] + + if isinstance(value, str): + text = value.strip() + return [ConversationMessage(role=default_role, content=text)] if text else [] + + if isinstance(value, dict): + # Common wrappers: {"messages": [...]}, {"input": ...}, {"output": ...} + for key in ("messages", "conversation", "chat"): + if isinstance(value.get(key), list): + return _messages_from_value(value[key], default_role) + + if "role" in value and "content" in value: + role = str(value.get("role") or default_role) + content = _content_to_text(value.get("content")).strip() + return [ConversationMessage(role=role, content=content, metadata={"source": "langfuse"})] if content else [] + + text = _content_to_text(value).strip() + return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else [] + + if isinstance(value, list): + out: list[ConversationMessage] = [] + for item in value: + out.extend(_messages_from_value(item, default_role)) + return out + + text = _content_to_text(value).strip() + return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else [] + + +def _agent_id(trace: dict[str, Any], detail: dict[str, Any] | None = None) -> str | None: + detail = detail or {} + meta = {**_metadata(trace), **_metadata(detail)} + return ( + meta.get("agent_id") + or meta.get("agentId") + or meta.get("agent") + or detail.get("name") + or trace.get("name") + ) + + +def _channel(trace: dict[str, Any], detail: dict[str, Any] | None = None) -> str | None: + detail = detail or {} + meta = {**_metadata(trace), **_metadata(detail)} + return meta.get("channel") or meta.get("channel_id") or meta.get("channelId") + + +def _observation_sort_key(obs: dict[str, Any]) -> str: + return str( + obs.get("startTime") + or obs.get("start_time") + or obs.get("createdAt") + or obs.get("created_at") + or obs.get("timestamp") + or "" + ) + + +class LangfuseCollector(ConversationCollector): + """Collect traces from Langfuse and hydrate each trace with detail/observations. + + The list endpoint often returns only trace metadata. If we judge that directly, + prompts reach the LLM as empty conversations and the judge correctly returns + "Conversa vazia"/"Resposta vazia". This collector therefore fetches each trace + detail and observations before building ConversationRecord. + """ + + def __init__(self): + self.identity_resolver = IdentityResolver(settings.identity_config_path) + + async def collect( + self, + period_start: datetime, + period_end: datetime, + agent_aliases: set[str] | None = None, + limit: int | None = None, + ) -> list[ConversationRecord]: + if not settings.can_use_langfuse: + raise RuntimeError( + "Langfuse disabled or credentials missing. Set ENABLE_LANGFUSE=true and LANGFUSE_PUBLIC_KEY/SECRET_KEY." + ) + + params = { + "fromTimestamp": _iso_z(period_start), + "toTimestamp": _iso_z(period_end), + "limit": limit or 100, + } + auth = (settings.langfuse_public_key, settings.langfuse_secret_key) + aliases = {a for a in (agent_aliases or set()) if a} + + async with httpx.AsyncClient(base_url=settings.langfuse_host, timeout=60) as client: + response = await client.get("/api/public/traces", params=params, auth=auth) + if response.status_code >= 400: + raise RuntimeError(f"Langfuse traces API failed {response.status_code}: {response.text}") + payload = response.json() + traces = payload.get("data") or payload.get("traces") or [] + + records: list[ConversationRecord] = [] + for trace in traces: + if not isinstance(trace, dict): + continue + trace_id = trace.get("id") + detail = await self._fetch_trace_detail(client, trace_id, auth) if trace_id else {} + observations = await self._fetch_observations(client, trace_id, auth) if trace_id else [] + + agent_id = _agent_id(trace, detail) + if aliases and agent_id and agent_id not in aliases: + continue + + record = self._to_record(trace, detail, observations, agent_id) + # Do not send empty traces to the LLM judge. Empty records produce + # valid but misleading scores such as "Conversa vazia". + if not (record.input_text or record.output_text or record.messages): + continue + records.append(record) + + return records + + async def _fetch_trace_detail( + self, + client: httpx.AsyncClient, + trace_id: str, + auth: tuple[str | None, str | None], + ) -> dict[str, Any]: + # Langfuse versions differ slightly. This endpoint works in current public API; + # if unavailable, the collector falls back to the list payload. + response = await client.get(f"/api/public/traces/{trace_id}", auth=auth) + if response.status_code >= 400: + return {} + payload = response.json() + if isinstance(payload, dict): + return payload.get("data") if isinstance(payload.get("data"), dict) else payload + return {} + + async def _fetch_observations( + self, + client: httpx.AsyncClient, + trace_id: str, + auth: tuple[str | None, str | None], + ) -> list[dict[str, Any]]: + # Try common Langfuse public API shapes. Ignore failures because trace detail + # may already contain observations in some versions. + candidates = [ + ("/api/public/observations", {"traceId": trace_id, "limit": 100}), + ("/api/public/observations", {"trace_id": trace_id, "limit": 100}), + ] + for path, params in candidates: + response = await client.get(path, params=params, auth=auth) + if response.status_code >= 400: + continue + payload = response.json() + items = payload.get("data") or payload.get("observations") or [] if isinstance(payload, dict) else [] + if isinstance(items, list): + return [x for x in items if isinstance(x, dict)] + return [] + + def _to_record( + self, + trace: dict[str, Any], + detail: dict[str, Any], + observations: list[dict[str, Any]], + agent_id: str | None, + ) -> ConversationRecord: + meta = {**_metadata(trace), **_metadata(detail)} + trace_id = trace.get("id") or detail.get("id") + session_id = ( + detail.get("sessionId") + or detail.get("session_id") + or trace.get("sessionId") + or trace.get("session_id") + or trace_id + ) + + # Detail payload may already include observations. + detail_observations = detail.get("observations") or [] + if isinstance(detail_observations, list): + observations = [*observations, *[x for x in detail_observations if isinstance(x, dict)]] + observations = sorted(observations, key=_observation_sort_key) + + input_value = _first_value( + detail.get("input"), + trace.get("input"), + meta.get("input"), + meta.get("user_message"), + meta.get("message"), + meta.get("question"), + ) + output_value = _first_value( + detail.get("output"), + trace.get("output"), + meta.get("output"), + meta.get("response"), + meta.get("answer"), + ) + + messages: list[ConversationMessage] = [] + messages.extend(_messages_from_value(input_value, "user")) + messages.extend(_messages_from_value(output_value, "assistant")) + + for obs in observations: + obs_meta = _metadata(obs) + obs_kind = str(obs.get("type") or obs.get("name") or "observation").lower() + source_meta = {"source": "langfuse_observation", "observation_id": obs.get("id"), "observation_type": obs_kind} + if obs_meta: + source_meta["metadata"] = obs_meta + + # Prefer preserving explicit chat roles from observation input. + before = len(messages) + messages.extend(_messages_from_value(obs.get("input"), "user")) + for m in messages[before:]: + m.metadata.update(source_meta) + + before = len(messages) + default_output_role = "assistant" if obs_kind in {"generation", "span", "event", "observation"} else "assistant" + messages.extend(_messages_from_value(obs.get("output"), default_output_role)) + for m in messages[before:]: + m.metadata.update(source_meta) + + messages = self._deduplicate_messages(messages) + + input_text = _content_to_text(input_value).strip() + output_text = _content_to_text(output_value).strip() + + if not input_text: + first_user = next((m.content for m in messages if m.role.lower() in {"user", "human"} and m.content), "") + input_text = first_user.strip() + if not output_text: + last_assistant = next( + (m.content for m in reversed(messages) if m.role.lower() in {"assistant", "agent", "ai"} and m.content), + "", + ) + output_text = last_assistant.strip() + + raw = {"trace": trace, "detail": detail, "observations": observations} + + identity_payload = { + **(trace.get("metadata") or {}), + **(trace.get("input") or {}), + "business_context": (trace.get("input") or {}).get("business_context") or {}, + "session_id": trace.get("sessionId") or trace.get("id"), + "message_id": (trace.get("input") or {}).get("message_id"), + "conversation_key": (trace.get("input") or {}).get("conversation_key"), + } + + business_context = self.identity_resolver.resolve(identity_payload) + + metadata = { + **(trace.get("metadata") or {}), + "business_context": business_context, + "ura_call_id": business_context.get("interaction_key"), + } + channel = ( + trace.get("metadata", {}).get("channel") + or trace.get("input", {}).get("channel") + or trace.get("input", {}).get("metadata", {}).get("channel") + or "web" + ) + + return ConversationRecord( + trace_id=trace_id, + session_id=business_context.get("session_key") or trace_id, + message_id=business_context.get("interaction_key") or trace_id, + agent_id=agent_id, + channel=channel, + input_text=input_text, + output_text=output_text, + messages=messages, + metadata=metadata, + raw=raw, + ) + + def _deduplicate_messages(self, messages: list[ConversationMessage]) -> list[ConversationMessage]: + out: list[ConversationMessage] = [] + seen: set[tuple[str, str]] = set() + for msg in messages: + content = (msg.content or "").strip() + if not content: + continue + key = (msg.role.lower(), content) + if key in seen: + continue + seen.add(key) + msg.content = content + out.append(msg) + return out diff --git a/evaluator/collectors/mock.py b/evaluator/collectors/mock.py new file mode 100644 index 0000000..d6efbe5 --- /dev/null +++ b/evaluator/collectors/mock.py @@ -0,0 +1,9 @@ +from __future__ import annotations +from datetime import datetime +from evaluator.collectors.base import ConversationCollector +from evaluator.core.models import ConversationRecord, ConversationMessage + +class MockCollector(ConversationCollector): + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None=None, limit: int | None=None): + agent = next(iter(agent_aliases), 'telecom_contas') if agent_aliases else 'telecom_contas' + return [ConversationRecord(trace_id='mock-trace-1', session_id='mock-session-1', message_id='mock-message-1', agent_id=agent, channel='web', input_text='quero minha fatura', output_text='Sua fatura está em aberto no valor de R$ 120.', messages=[ConversationMessage(role='user', content='quero minha fatura'), ConversationMessage(role='assistant', content='Sua fatura está em aberto no valor de R$ 120.')], metadata={'mock': True})] diff --git a/evaluator/config/.DS_Store b/evaluator/config/.DS_Store new file mode 100644 index 0000000..bb02fe0 Binary files /dev/null and b/evaluator/config/.DS_Store differ diff --git a/evaluator/config/__pycache__/agents.cpython-313.pyc b/evaluator/config/__pycache__/agents.cpython-313.pyc new file mode 100644 index 0000000..ad2dae3 Binary files /dev/null and b/evaluator/config/__pycache__/agents.cpython-313.pyc differ diff --git a/evaluator/config/__pycache__/settings.cpython-313.pyc b/evaluator/config/__pycache__/settings.cpython-313.pyc new file mode 100644 index 0000000..d385d17 Binary files /dev/null and b/evaluator/config/__pycache__/settings.cpython-313.pyc differ diff --git a/evaluator/config/agents.py b/evaluator/config/agents.py new file mode 100644 index 0000000..c080a6a --- /dev/null +++ b/evaluator/config/agents.py @@ -0,0 +1,36 @@ +from __future__ import annotations +from dataclasses import dataclass, field +import yaml +from pathlib import Path +from evaluator.config.settings import settings + +@dataclass +class AgentConfig: + agent_id: str + enabled: bool = True + days_back: int = 1 + percentage: float = 1.0 + langfuse_agent_aliases: list[str] = field(default_factory=list) + gcs_prefix: str = "" + + @property + def aliases(self) -> set[str]: + return {self.agent_id, *self.langfuse_agent_aliases} + + +def load_agents(path: str | None = None) -> list[AgentConfig]: + p = settings.path(path or settings.agents_config_path) + data = yaml.safe_load(p.read_text()) or {} + agents = [] + for item in data.get("agents", []): + cfg = AgentConfig( + agent_id=item["agent_id"], + enabled=bool(item.get("enabled", True)), + days_back=int(item.get("days_back", item.get("daysBack", 1))), + percentage=float(item.get("percentage", 1.0)), + langfuse_agent_aliases=list(item.get("langfuse_agent_aliases", [])), + gcs_prefix=str(item.get("gcs_prefix", "")), + ) + if cfg.enabled: + agents.append(cfg) + return agents diff --git a/evaluator/config/settings.py b/evaluator/config/settings.py new file mode 100644 index 0000000..ce60c4f --- /dev/null +++ b/evaluator/config/settings.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import os +from pathlib import Path +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict +from dotenv import load_dotenv + +ROOT_DIR = Path(__file__).resolve().parents[2] +ENV_PATH = ROOT_DIR / ".env" +load_dotenv(ENV_PATH, override=True) + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=str(ENV_PATH), extra="ignore", case_sensitive=False) + + adb_user: str = Field(default="", validation_alias="ADB_USER") + adb_password: str = Field(default="", validation_alias="ADB_PASSWORD") + adb_dsn: str = Field(default="", validation_alias="ADB_DSN") + adb_wallet_location: str | None = Field(default=None, validation_alias="ADB_WALLET_LOCATION") + adb_wallet_password: str | None = Field(default=None, validation_alias="ADB_WALLET_PASSWORD") + adb_table_prefix: str = Field(default="AGENTFW", validation_alias="ADB_TABLE_PREFIX") + + enable_langfuse: bool = Field(default=False, validation_alias="ENABLE_LANGFUSE") + langfuse_public_key: str | None = Field(default=None, validation_alias="LANGFUSE_PUBLIC_KEY") + langfuse_secret_key: str | None = Field(default=None, validation_alias="LANGFUSE_SECRET_KEY") + langfuse_host: str = Field(default="http://localhost:3005", validation_alias="LANGFUSE_HOST") + publish_langfuse_scores: bool = Field(default=False, validation_alias="PUBLISH_LANGFUSE_SCORES") + + # llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER") + # llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE") + # llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH") + # oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT") + # oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID") + # oci_genai_compartment_id: str | None = Field(default=None, validation_alias="OCI_GENAI_COMPARTMENT_ID") + # oci_genai_auth_type: str = Field(default="api_key", validation_alias="OCI_GENAI_AUTH_TYPE") + # oci_config_path: str | None = Field(default=None, validation_alias="OCI_CONFIG_PATH") + # oci_config_profile: str = Field(default="DEFAULT", validation_alias="OCI_CONFIG_PROFILE") + # llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE") + # llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS") + + # LLM / OCI GenAI OpenAI-compatible, mesmo padrão do Agent Framework + llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER") + llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE") + llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH") + + oci_genai_base_url: str | None = Field(default=None, validation_alias="OCI_GENAI_BASE_URL") + oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT") # compatibilidade + oci_genai_model: str = Field(default="openai.gpt-4.1", validation_alias="OCI_GENAI_MODEL") + oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID") # compatibilidade + oci_genai_api_key: str | None = Field(default=None, validation_alias="OCI_GENAI_API_KEY") + oci_genai_project_ocid: str | None = Field(default=None, validation_alias="OCI_GENAI_PROJECT_OCID") + + llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE") + llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS") + llm_timeout_seconds: int = Field(default=120, validation_alias="LLM_TIMEOUT_SECONDS") + + agents_config_path: str = Field(default="configs/judge/agents.yaml", validation_alias="AGENTS_CONFIG_PATH") + trace_prompt_path: str = Field(default="configs/judge/trace_metrics.yaml", validation_alias="TRACE_PROMPT_PATH") + session_prompt_path: str = Field(default="configs/judge/session_metrics.yaml", validation_alias="SESSION_PROMPT_PATH") + output_dir: str = Field(default="output", validation_alias="OUTPUT_DIR") + batch_size: int = Field(default=50, validation_alias="BATCH_SIZE") + max_attempts: int = Field(default=3, validation_alias="MAX_ATTEMPTS") + enable_gcs_upload: bool = Field(default=False, validation_alias="ENABLE_GCS_UPLOAD") + judge_gcs_bucket: str | None = Field(default=None, validation_alias="JUDGE_GCS_BUCKET") + google_application_credentials: str | None = Field(default=None, validation_alias="GOOGLE_APPLICATION_CREDENTIALS") + identity_config_path: str = "configs/identity.yaml" + + @property + def project_root(self) -> Path: + return ROOT_DIR + + def path(self, value: str | Path) -> Path: + p = Path(value) + return p if p.is_absolute() else ROOT_DIR / p + + @property + def ADB_USER(self): return self.adb_user + @property + def ADB_PASSWORD(self): return self.adb_password + @property + def ADB_DSN(self): return self.adb_dsn + @property + def ADB_WALLET_LOCATION(self): return self.adb_wallet_location + @property + def ADB_WALLET_PASSWORD(self): return self.adb_wallet_password + @property + def ADB_TABLE_PREFIX(self): return (self.adb_table_prefix or "AGENTFW").upper().rstrip("_") + + @property + def has_langfuse_credentials(self) -> bool: + return bool(self.langfuse_public_key and self.langfuse_secret_key) + + @property + def can_use_langfuse(self) -> bool: + return bool(self.enable_langfuse and self.has_langfuse_credentials) + + @property + def can_publish_langfuse_scores(self) -> bool: + return bool(self.publish_langfuse_scores and self.can_use_langfuse) + + @property + def OCI_GENAI_BASE_URL(self) -> str | None: + return self.oci_genai_base_url or self.oci_genai_endpoint + + @property + def OCI_GENAI_MODEL(self) -> str: + return self.oci_genai_model_id or self.oci_genai_model + + @property + def OCI_GENAI_API_KEY(self) -> str | None: + return self.oci_genai_api_key + + @property + def OCI_GENAI_PROJECT_OCID(self) -> str | None: + return self.oci_genai_project_ocid + + @property + def LLM_PROVIDER(self) -> str: + return self.llm_provider + + @property + def LLM_TEMPERATURE(self) -> float: + return self.llm_temperature + + @property + def LLM_MAX_TOKENS(self) -> int: + return self.llm_max_tokens + + @property + def LLM_TIMEOUT_SECONDS(self) -> int: + return self.llm_timeout_seconds + + @property + def LLM_PROFILES_PATH(self) -> str: + return self.llm_profiles_path + +settings = Settings() +if settings.ADB_WALLET_LOCATION: + os.environ["TNS_ADMIN"] = settings.ADB_WALLET_LOCATION diff --git a/evaluator/core/__pycache__/models.cpython-313.pyc b/evaluator/core/__pycache__/models.cpython-313.pyc new file mode 100644 index 0000000..c3e2b71 Binary files /dev/null and b/evaluator/core/__pycache__/models.cpython-313.pyc differ diff --git a/evaluator/core/models.py b/evaluator/core/models.py new file mode 100644 index 0000000..f76e448 --- /dev/null +++ b/evaluator/core/models.py @@ -0,0 +1,56 @@ +from __future__ import annotations +from datetime import datetime +from enum import Enum +from pydantic import BaseModel, Field +from typing import Any + +class RunStatus(str, Enum): + RUNNING = "RUNNING" + COMPLETED = "COMPLETED" + PARTIAL = "PARTIAL" + FAILED = "FAILED" + +class ItemStatus(str, Enum): + PENDING = "PENDING" + PROCESSING = "PROCESSING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + SKIPPED = "SKIPPED" + +class ConversationMessage(BaseModel): + role: str + content: str + created_at: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + +class ConversationRecord(BaseModel): + trace_id: str | None = None + session_id: str + message_id: str | None = None + agent_id: str | None = None + channel: str | None = None + input_text: str = "" + output_text: str = "" + messages: list[ConversationMessage] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + raw: dict[str, Any] = Field(default_factory=dict) + +class TraceJudgeResult(BaseModel): + judgeScore: float + accuracyScore: float + alucinationScore: float + rationale: str = "" + judge_name: str = "trace_metrics" + judge_type: str = "trace" + +class SessionJudgeResult(BaseModel): + inferredCsiScore: float + resolution: int + conversationPrecision: int + rationale: str = "" + judge_name: str = "session_metrics" + judge_type: str = "session" + +class CombinedJudgeResult(BaseModel): + trace: TraceJudgeResult + session: SessionJudgeResult | None = None diff --git a/evaluator/engine.py b/evaluator/engine.py new file mode 100644 index 0000000..3abf3b7 --- /dev/null +++ b/evaluator/engine.py @@ -0,0 +1,144 @@ +from __future__ import annotations +import inspect, json, random +from datetime import datetime, timedelta +from typing import Any, Awaitable, Callable +from evaluator.collectors.base import ConversationCollector +from evaluator.collectors.langfuse import LangfuseCollector +from evaluator.collectors.agent_framework import AgentFrameworkCollector +from evaluator.collectors.mock import MockCollector +from evaluator.config.agents import AgentConfig +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, RunStatus +from evaluator.judges.llm_judge import TIMStyleLLMJudge +from evaluator.output.legacy_exporter import export_legacy_txt_gz +from evaluator.persistence.repository import EvaluationRepository +from evaluator.publishers.langfuse_scores import LangfuseScorePublisher + +ProgressCallback = Callable[[dict[str, Any]], Awaitable[None] | None] + +class EvaluationEngine: + def __init__(self, repository: EvaluationRepository | None=None, progress_callback: ProgressCallback | None=None): + self.repository = repository or EvaluationRepository(auto_init_schema=False) + self.progress_callback = progress_callback + self.judge = TIMStyleLLMJudge() + self.langfuse_publisher = LangfuseScorePublisher() + + # async def _emit(self, run_id: str, stage: str, message: str='', **details): + # details.pop('run_id', None) + # await self.repository.arecord_progress(run_id, stage, message, details) + # event={'run_id': run_id, 'stage': stage, 'message': message, 'details': details} + async def _emit(self, progress_run_id: str, stage: str, message: str = "", **details): + details.pop("run_id", None) + + await self.repository.arecord_progress( + progress_run_id, + stage, + message, + details, + ) + + event = { + "run_id": progress_run_id, + "stage": stage, + "message": message, + "details": details, + } + + if self.progress_callback: + r = self.progress_callback(event) + if inspect.isawaitable(r): await r + + def collector_for(self, source: str) -> ConversationCollector: + if source == 'langfuse': return LangfuseCollector() + if source == 'agent_framework': return AgentFrameworkCollector() + if source == 'mock': return MockCollector() + raise ValueError('source must be langfuse, agent_framework or mock') + + async def run_agent(self, agent: AgentConfig, period_start: datetime, period_end: datetime, source: str='langfuse', limit: int | None=None) -> dict: + run_id = await self.repository.acreate_run(period_start, period_end, source, agent.agent_id) + try: + await self._emit(run_id, 'RUN_CREATED', f'Agent run created: {agent.agent_id}', agent_id=agent.agent_id, source=source) + collector = self.collector_for(source) + await self._emit(run_id, 'COLLECTING', 'Collecting conversations') + records = await collector.collect(period_start, period_end, agent_aliases=agent.aliases, limit=limit) + await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} records before sampling') + records = self._sample(records, agent.percentage) + await self._emit(run_id, 'SAMPLED', f'Kept {len(records)} records', percentage=agent.percentage) + inserted = await self.repository.ainsert_items(run_id, records) + await self._emit(run_id, 'ITEMS_INSERTED', f'Inserted {inserted} items') + summary = await self._process(run_id) + output_path = export_legacy_txt_gz(self.repository, run_id, agent.agent_id) + await self._emit(run_id, 'EXPORTED', f'Exported {output_path}', output_file=str(output_path)) + return {**summary, 'agent_id': agent.agent_id, 'output_file': str(output_path), 'uploaded_to': None} + except Exception as exc: + await self.repository.amark_run_status(run_id, RunStatus.PARTIAL, str(exc)) + await self._emit(run_id, 'PARTIAL', f'Run failed: {exc}', error=str(exc)) + return {'status':'PARTIAL','run_id':run_id,'agent_id':agent.agent_id,'error':str(exc)} + + async def run(self, period_start: datetime, period_end: datetime, source: str='langfuse', limit: int | None=None) -> dict: + run_id = await self.repository.acreate_run(period_start, period_end, source, None) + try: + collector = self.collector_for(source) + await self._emit(run_id, 'COLLECTING', 'Collecting conversations') + records = await collector.collect(period_start, period_end, limit=limit) + await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} records') + await self.repository.ainsert_items(run_id, records) + return await self._process(run_id) + except Exception as exc: + await self.repository.amark_run_status(run_id, RunStatus.PARTIAL, str(exc)) + await self._emit(run_id, 'PARTIAL', f'Run failed: {exc}', error=str(exc)) + return {'status':'PARTIAL','run_id':run_id,'error':str(exc)} + + async def _process(self, run_id: str) -> dict: + processed_records: list[ConversationRecord] = [] + while True: + items = await self.repository.afetch_next_items(run_id, settings.batch_size) + if not items: break + await self._emit(run_id, 'BATCH_STARTED', f'Processing {len(items)} items') + for item in items: + item_id=item['item_id'] + await self.repository.amark_item_processing(item_id) + try: + raw=item['raw_json'] + if hasattr(raw, 'read'): raw = raw.read() + record = ConversationRecord.model_validate(json.loads(raw)) + result = await self.judge.judge_trace(record) + await self.repository.asave_trace_result(run_id, item_id, record, result) + await self.langfuse_publisher.publish_trace_score(record, result) + await self.repository.amark_item_completed(run_id, item_id) + processed_records.append(record) + + #await self._emit(run_id, 'ITEM_COMPLETED', f'Item completed {item_id}', trace_id=record.trace_id) + loop_result = getattr(result, "loop_result", None) + + await self._emit( + run_id, + "ITEM_COMPLETED", + f"Item completed {item_id}", + trace_id=record.trace_id, + session_id=record.session_id, + judgeScore=result.judgeScore, + accuracyScore=result.accuracyScore, + alucinationScore=result.alucinationScore, + rationale=result.rationale, + loop=getattr(loop_result, "loop", 0) if loop_result else 0, + loop_reason=getattr(loop_result, "reason", "") if loop_result else "", + ) + except Exception as exc: + await self.repository.amark_item_failed(run_id, item_id, str(exc)) + await self._emit(run_id, 'ITEM_FAILED', f'Item failed {item_id}', error=str(exc)) + if processed_records: + sessions = await self.judge.judge_sessions(processed_records) + for sid, result in sessions.items(): + agent_id = next((r.agent_id for r in processed_records if r.session_id == sid), None) + await self.repository.asave_session_result(run_id, sid, agent_id, result) + await self._emit(run_id, 'SESSION_JUDGE_COMPLETED', f'Evaluated {len(sessions)} sessions') + await self.repository.amark_run_status(run_id, RunStatus.COMPLETED) + summary = await self.repository.asummarize_run(run_id) + await self._emit(run_id, 'COMPLETED', 'Run completed', **summary) + return {'status':'COMPLETED', **summary} + + def _sample(self, records: list[ConversationRecord], percentage: float) -> list[ConversationRecord]: + if percentage >= 1: return records + rng = random.Random(42) + return [r for r in records if rng.random() <= percentage] diff --git a/evaluator/identity/__pycache__/resolver.cpython-313.pyc b/evaluator/identity/__pycache__/resolver.cpython-313.pyc new file mode 100644 index 0000000..6e1c106 Binary files /dev/null and b/evaluator/identity/__pycache__/resolver.cpython-313.pyc differ diff --git a/evaluator/identity/resolver.py b/evaluator/identity/resolver.py new file mode 100644 index 0000000..a0ea2e2 --- /dev/null +++ b/evaluator/identity/resolver.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml + + +def _deep_get(data: dict, path: str): + cur = data + for part in path.split("."): + if not isinstance(cur, dict): + return None + cur = cur.get(part) + return cur + + +class IdentityResolver: + def __init__(self, path: str = "configs/identity.yaml"): + self.path = Path(path) + self.config = yaml.safe_load(self.path.read_text(encoding="utf-8")) or {} + self.identity = self.config.get("identity", {}) + self.keys = self.identity.get("keys", {}) + + def resolve(self, payload: dict[str, Any]) -> dict[str, Any]: + out = {} + + for key, spec in self.keys.items(): + value = None + for source in spec.get("sources", []): + value = _deep_get(payload, source) + if value not in (None, ""): + break + out[key] = str(value) if value not in (None, "") else None + + out["metadata"] = { + "identity_version": self.identity.get("version"), + "identity_source": str(self.path), + } + + return out \ No newline at end of file diff --git a/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc b/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc new file mode 100644 index 0000000..9613a01 Binary files /dev/null and b/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc differ diff --git a/evaluator/judges/llm_judge.py b/evaluator/judges/llm_judge.py new file mode 100644 index 0000000..c69e77e --- /dev/null +++ b/evaluator/judges/llm_judge.py @@ -0,0 +1,81 @@ +from __future__ import annotations +import json +import re +from collections import defaultdict +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, TraceJudgeResult, SessionJudgeResult +from evaluator.llm.client import LLMClient, create_llm_client +from evaluator.prompts.loader import load_prompt + + +def _json_from_text(text: str) -> dict: + try: + return json.loads(text) + except Exception: + m = re.search(r"\{.*\}", text, flags=re.S) + if not m: + raise + return json.loads(m.group(0)) + + +def _history(record: ConversationRecord, max_chars: int = 6000) -> str: + if record.messages: + text = "\n".join(f"{m.role}: {m.content}" for m in record.messages) + else: + text = f"user: {record.input_text}\nagent: {record.output_text}" + return text[-max_chars:] + + +class TIMStyleLLMJudge: + def __init__(self, llm: LLMClient | None = None): + self.llm = llm or create_llm_client() + self.trace_prompt = load_prompt(settings.trace_prompt_path, 'trace_metrics') + self.session_prompt = load_prompt(settings.session_prompt_path, 'session_metrics') + + async def judge_trace(self, record: ConversationRecord) -> TraceJudgeResult: + prompt = f"""{self.trace_prompt} + +HISTÓRICO: +{_history(record)} + +MENSAGEM DO USUÁRIO: +{record.input_text} + +RESPOSTA DO AGENTE: +{record.output_text} + +METADATA: +{json.dumps(record.metadata, ensure_ascii=False, default=str)} +""" + raw = await self.llm.complete(prompt) + data = _json_from_text(raw) + data.setdefault("judge_name", "trace_metrics") + data.setdefault("judge_type", "trace") + data.setdefault("judgeScore", data.get("judge_score", 0)) + data.setdefault("accuracyScore", data.get("accuracy_score", 0)) + data.setdefault("alucinationScore", data.get("alucination_score", 1)) + data.setdefault("rationale", data.get("reasoning", "")) + return TraceJudgeResult(**data) + + async def judge_sessions(self, records: list[ConversationRecord]) -> dict[str, SessionJudgeResult]: + grouped: dict[str, list[ConversationRecord]] = defaultdict(list) + for r in records: + grouped[r.session_id].append(r) + out = {} + for session_id, items in grouped.items(): + transcript = "\n".join(_history(r, 3000) for r in items)[-9000:] + prompt = f"""{self.session_prompt} + +TRANSCRIÇÃO DA SESSÃO: +{transcript} +""" + raw = await self.llm.complete(prompt) + data = _json_from_text(raw) + data.setdefault("judge_name", "session_metrics") + data.setdefault("judge_type", "session") + data.setdefault("inferredCsiScore", data.get("inferred_csi_score", 0)) + data.setdefault("resolution", data.get("resolution", 0)) + data.setdefault("conversationPrecision", data.get("conversation_precision", 0)) + data.setdefault("rationale", data.get("reasoning", "")) + out[session_id] = SessionJudgeResult(**data) + return out diff --git a/evaluator/llm/__pycache__/client.cpython-313.pyc b/evaluator/llm/__pycache__/client.cpython-313.pyc new file mode 100644 index 0000000..4f33636 Binary files /dev/null and b/evaluator/llm/__pycache__/client.cpython-313.pyc differ diff --git a/evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc b/evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc new file mode 100644 index 0000000..92fd6ee Binary files /dev/null and b/evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc differ diff --git a/evaluator/llm/client.py b/evaluator/llm/client.py new file mode 100644 index 0000000..11ae184 --- /dev/null +++ b/evaluator/llm/client.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import json +from typing import Any + +from evaluator.config.settings import settings +from evaluator.llm.profile_resolver import LLMProfileResolver + + +class LLMClient: + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + raise NotImplementedError + + +class MockLLMClient(LLMClient): + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + if "inferredCsiScore" in prompt: + return json.dumps({ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "Avaliação mock." + }, ensure_ascii=False) + + return json.dumps({ + "judgeScore": 0.7, + "accuracyScore": 0.7, + "alucinationScore": 0.1, + "rationale": "Avaliação mock." + }, ensure_ascii=False) + + +class OCICompatibleLLMClient(LLMClient): + """ + Mesmo padrão do Agent Framework: + - LLM_PROVIDER=oci_openai + - OCI_GENAI_BASE_URL + - OCI_GENAI_API_KEY + - OCI_GENAI_MODEL + - llm_profiles.yaml opcional + """ + + def __init__(self): + self.resolver = LLMProfileResolver(settings) + + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + effective = self.resolver.resolve(profile_name or settings.llm_profile) + + provider = str(effective.get("provider") or settings.LLM_PROVIDER) + model = str(effective.get("model") or settings.OCI_GENAI_MODEL) + base_url = effective.get("base_url") or settings.OCI_GENAI_BASE_URL + api_key = effective.get("api_key") or settings.OCI_GENAI_API_KEY + temperature = effective.get("temperature", settings.LLM_TEMPERATURE) + max_tokens = effective.get("max_tokens", settings.LLM_MAX_TOKENS) + timeout = effective.get("timeout_seconds", settings.LLM_TIMEOUT_SECONDS) + + if provider == "mock": + return await MockLLMClient().complete(prompt, profile_name=profile_name) + + if provider not in ("oci_openai", "openai_compatible"): + raise ValueError(f"Unsupported LLM provider: {provider}") + + if not base_url: + raise RuntimeError("OCI_GENAI_BASE_URL is required for oci_openai provider") + + if not api_key: + raise RuntimeError("OCI_GENAI_API_KEY is required for oci_openai provider") + + from openai import AsyncOpenAI + + client = AsyncOpenAI( + base_url=base_url, + api_key=api_key, + timeout=timeout, + ) + + resp = await client.chat.completions.create( + model=model, + messages=[ + {"role": "user", "content": prompt} + ], + temperature=temperature, + max_tokens=max_tokens, + ) + + return resp.choices[0].message.content or "" + + +def create_llm_client() -> LLMClient: + provider = (settings.LLM_PROVIDER or "mock").lower() + + if provider in ("mock", "none"): + return MockLLMClient() + + if provider in ("oci_openai", "openai_compatible", "oci"): + return OCICompatibleLLMClient() + + raise ValueError(f"Unsupported LLM_PROVIDER={settings.LLM_PROVIDER}") \ No newline at end of file diff --git a/evaluator/llm/profile_resolver.py b/evaluator/llm/profile_resolver.py new file mode 100644 index 0000000..06e5f26 --- /dev/null +++ b/evaluator/llm/profile_resolver.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import copy +import os +import re +from pathlib import Path +from typing import Any + +import yaml + + +def _canonical_profile_name(value: str | None) -> str: + name = (value or "default").strip() + name = name.replace("-", "_").replace(".", "_").replace(" ", "_") + name = re.sub(r"(? Path | None: + root = getattr(self.settings, "project_root", Path.cwd()) + configured = Path(getattr(self.settings, "LLM_PROFILES_PATH", "") or "").expanduser() + candidates = [] + if configured: + candidates.append(configured if configured.is_absolute() else Path(root) / configured) + candidates += [ + Path(root) / "llm_profiles.yaml", + Path(root) / "configs/llm_profiles/llm_profiles.yaml", + Path(root) / "config/llm_profiles.yaml", + Path("llm_profiles.yaml"), + Path("configs/llm_profiles/llm_profiles.yaml"), + ] + for path in candidates: + if path and path.exists() and path.is_file(): + return path + return None + + def _load_profiles(self, path: Path) -> dict[str, dict[str, Any]]: + # Expand ${VAR} placeholders, matching the way env-driven framework config is commonly used. + text = os.path.expandvars(path.read_text(encoding="utf-8")) + data = yaml.safe_load(text) or {} + raw = data.get("profiles", data) + profiles = {} + for name, value in raw.items(): + if isinstance(value, dict): + profiles[_canonical_profile_name(str(name))] = dict(value) + return profiles + + def env_defaults(self) -> dict[str, Any]: + return { + "provider": getattr(self.settings, "LLM_PROVIDER", "mock"), + "model": getattr(self.settings, "OCI_GENAI_MODEL", "mock-llm"), + "temperature": getattr(self.settings, "LLM_TEMPERATURE", 0.0), + "max_tokens": getattr(self.settings, "LLM_MAX_TOKENS", 900), + "timeout_seconds": getattr(self.settings, "LLM_TIMEOUT_SECONDS", 120), + "base_url": getattr(self.settings, "OCI_GENAI_BASE_URL", None), + "api_key": getattr(self.settings, "OCI_GENAI_API_KEY", None), + "project_ocid": getattr(self.settings, "OCI_GENAI_PROJECT_OCID", None), + } + + def resolve(self, profile_name: str | None = None, **overrides) -> dict[str, Any]: + profile_key = _canonical_profile_name(profile_name) + effective = self.env_defaults() + + if self.enabled: + effective.update(copy.deepcopy(self._profiles.get("default") or {})) + effective.update(copy.deepcopy(self._profiles.get(profile_key) or {})) + + for key, value in overrides.items(): + if value is not None: + effective[key] = value + + effective["profile_name"] = profile_key + return effective diff --git a/evaluator/output/.DS_Store b/evaluator/output/.DS_Store new file mode 100644 index 0000000..b72e607 Binary files /dev/null and b/evaluator/output/.DS_Store differ diff --git a/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc b/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc new file mode 100644 index 0000000..a170981 Binary files /dev/null and b/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc differ diff --git a/evaluator/output/legacy_exporter.py b/evaluator/output/legacy_exporter.py new file mode 100644 index 0000000..7fb3824 --- /dev/null +++ b/evaluator/output/legacy_exporter.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import gzip +from pathlib import Path +from datetime import datetime +from typing import Any +import json + +from evaluator.config.settings import settings +from evaluator.persistence.repository import EvaluationRepository +from evaluator.analytics.vloop import vloop_flag + +HEADER = [ + "judgeScore", "accuracyScore", "alucinationScore", "promptLength", "loop", + "inferredCsiScore", "resolution", "conversationPrecision", + "uraCallId", "channelId", "sessionId", "messageId" +] + + +def _q(v) -> str: + return '"' + str("" if v is None else v).replace('"', '""') + '"' + + +def export_legacy_txt_gz(repo: EvaluationRepository, run_id: str, agent_id: str) -> Path: + output_dir = settings.path(settings.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / f"AGENTE_{agent_id}_LLM_JUDGE_{datetime.now().strftime('%Y%m%d')}.TXT.GZ" + + with repo.store.connect() as conn: + cur = conn.cursor() + + cur.execute(f""" + select SESSION_ID, INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION + from {repo.store.t('EVALUATION_RESULT')} + where RUN_ID = :run_id + and JUDGE_TYPE = 'SESSION' + """, {"run_id": run_id}) + + session_metrics = { + sid: { + "inferredCsiScore": csi, + "resolution": res, + "conversationPrecision": prec, + } + for sid, csi, res, prec in cur.fetchall() + } + + cur.execute(f""" + select r.TRACE_ID, r.SESSION_ID, r.JUDGE_SCORE, r.ACCURACY_SCORE, + r.ALUCINATION_SCORE, r.RATIONALE, i.CHANNEL, i.MESSAGE_ID, i.RAW_JSON + from {repo.store.t('EVALUATION_RESULT')} r + left join {repo.store.t('EVALUATION_ITEM')} i on i.ITEM_ID = r.ITEM_ID + where r.RUN_ID = :run_id + and r.JUDGE_TYPE = 'TRACE' + order by r.CREATED_AT + """, {"run_id": run_id}) + + rows = cur.fetchall() + + with gzip.open(path, "wt", encoding="utf-8") as f: + for trace_id, session_id, judge, accuracy, alucination, rationale, channel, message_id, raw_json in rows: + session = session_metrics.get(session_id, {}) + + raw: dict[str, Any] = {} + ura_call_id = "" + channel_id = channel or "" + prompt_length = 0 + loop = 0 + + try: + from evaluator.persistence.oracle_store import _json_loads + + # raw = _json_loads( + # raw_json.read() if hasattr(raw_json, "read") else raw_json, + # {}, + # ) + raw = normalize_raw(raw_json) + + metadata = raw.get("metadata") or {} + + channel_id = ( + metadata.get("channel_id") + or metadata.get("channelId") + or metadata.get("channel") + or channel_id + ) + + ura_call_id = extract_ura_call_id(raw, metadata, message_id) + prompt_length = extract_prompt_length(raw) + loop = vloop_flag(raw) + + # print( + # "[DEBUG promptLength]", + # "trace_id=", trace_id, + # "type(raw)=", type(raw), + # "keys=", list(raw.keys())[:20] if isinstance(raw, dict) else None, + # "prompt_length=", prompt_length, + # "input_text_len=", len(str(raw.get("input_text") or "")) if isinstance(raw, dict) else None, + # "messages=", len(raw.get("messages") or []) if isinstance(raw, dict) else None, + # ) + + except Exception as exc: + print(f"[legacy_exporter] metadata extraction failed trace_id={trace_id}: {exc}") + + vals = [ + judge, + accuracy, + alucination, + prompt_length, + loop, + session.get("inferredCsiScore"), + session.get("resolution"), + session.get("conversationPrecision"), + ura_call_id, + channel_id, + session_id, + message_id or trace_id, + ] + + f.write("|;".join(_q(v) for v in vals) + "\n") + + f.write("|;".join([_q("TOTAL"), _q(len(rows))]) + "\n") + + return path + +def extract_ura_call_id(raw: dict, metadata: dict | None = None, message_id: str | None = None) -> str: + metadata = metadata or {} + + business_context = ( + metadata.get("business_context") + or metadata.get("businessContext") + or raw.get("business_context") + or raw.get("businessContext") + or raw.get("metadata", {}).get("business_context") + or raw.get("metadata", {}).get("businessContext") + or {} + ) + + if not isinstance(business_context, dict): + business_context = {} + + trace = raw.get("raw", {}).get("trace", {}) or raw.get("trace", {}) or {} + detail = raw.get("raw", {}).get("detail", {}) or raw.get("detail", {}) or {} + + trace_input = trace.get("input") or {} + detail_input = detail.get("input") or {} + + trace_metadata = trace.get("metadata") or {} + detail_metadata = detail.get("metadata") or {} + + trace_bc = trace_input.get("business_context") or {} + detail_bc = detail_input.get("business_context") or {} + + return str( + business_context.get("interaction_key") + or business_context.get("ura_call_id") + or metadata.get("ura_call_id") + or metadata.get("uraCallId") + or metadata.get("interaction_key") + or trace_metadata.get("ura_call_id") + or detail_metadata.get("ura_call_id") + or trace_bc.get("interaction_key") + or detail_bc.get("interaction_key") + or message_id + or "" + ) + +def normalize_raw(raw): + if hasattr(raw, "read"): + raw = raw.read() + + if isinstance(raw, bytes): + raw = raw.decode("utf-8") + + if isinstance(raw, str): + raw = raw.strip() + if not raw: + return {} + raw = json.loads(raw) + + # caso esteja duplamente serializado + if isinstance(raw, str): + raw = json.loads(raw) + + return raw if isinstance(raw, dict) else {} + +def extract_prompt_length(raw: dict) -> int: + # 1. tokens reais do Langfuse/framework + tokens = find_prompt_tokens(raw) + if tokens > 0: + return tokens + + # 2. input_size dos spans + input_size = find_input_size(raw) + if input_size > 0: + return input_size + + # 3. fallback garantido pelo ConversationRecord + return ( + len(str(raw.get("input_text") or "")) + + len(str(raw.get("output_text") or "")) + + sum( + len(str(m.get("content") or "")) + for m in raw.get("messages", []) + if isinstance(m, dict) + ) + ) + +def _walk(obj): + if isinstance(obj, dict): + yield obj + for value in obj.values(): + yield from _walk(value) + elif isinstance(obj, list): + for item in obj: + yield from _walk(item) + + +def _to_positive_int(value) -> int: + try: + n = int(value) + return n if n > 0 else 0 + except Exception: + return 0 + + +def find_prompt_tokens(raw: dict) -> int: + candidates = [] + + for obj in _walk(raw): + for key in ( + "prompt_tokens", + "promptTokens", + "input_tokens", + "inputTokens", + ): + n = _to_positive_int(obj.get(key)) + if n: + candidates.append(n) + + usage = obj.get("usage") + if isinstance(usage, dict): + for key in ("input", "prompt_tokens", "promptTokens", "input_tokens", "inputTokens"): + n = _to_positive_int(usage.get(key)) + if n: + candidates.append(n) + + usage_details = obj.get("usageDetails") or obj.get("usage_details") + if isinstance(usage_details, dict): + for key in ("input", "prompt_tokens", "promptTokens", "input_tokens", "inputTokens"): + n = _to_positive_int(usage_details.get(key)) + if n: + candidates.append(n) + + return max(candidates) if candidates else 0 + + +def find_input_size(raw: dict) -> int: + candidates = [] + + for obj in _walk(raw): + for key in ("input_size", "inputSize"): + n = _to_positive_int(obj.get(key)) + if n: + candidates.append(n) + + return max(candidates) if candidates else 0 + +def calculate_text_length(raw: dict) -> int: + return ( + len(str(raw.get("input_text") or "")) + + len(str(raw.get("output_text") or "")) + + sum( + len(str(m.get("content") or "")) + for m in raw.get("messages", []) + if isinstance(m, dict) + ) + ) \ No newline at end of file diff --git a/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc b/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc new file mode 100644 index 0000000..1698425 Binary files /dev/null and b/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc differ diff --git a/evaluator/persistence/__pycache__/repository.cpython-313.pyc b/evaluator/persistence/__pycache__/repository.cpython-313.pyc new file mode 100644 index 0000000..166797e Binary files /dev/null and b/evaluator/persistence/__pycache__/repository.cpython-313.pyc differ diff --git a/evaluator/persistence/oracle_store.py b/evaluator/persistence/oracle_store.py new file mode 100644 index 0000000..fa2d97c --- /dev/null +++ b/evaluator/persistence/oracle_store.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import asyncio +import json +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any + + +def _json_dumps(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _json_loads(value: str | bytes | None, default: Any): + if value is None: + return default + if isinstance(value, bytes): + value = value.decode("utf-8") + try: + return json.loads(value) + except Exception: + return default + + +@dataclass +class OracleSettings: + user: str + password: str + dsn: str + wallet_location: str | None = None + wallet_password: str | None = None + table_prefix: str = "AGENTFW" + + +class OracleStore: + """Oracle Autonomous Database store following the Agent Framework pattern. + + Uses direct oracledb.connect() with wallet arguments. Synchronous DB calls are + exposed through asyncio.to_thread(), matching the main framework style. + """ + + def __init__(self, settings, auto_init_schema: bool = False): + self.settings = settings + self.cfg = OracleSettings( + user=settings.ADB_USER or "", + password=settings.ADB_PASSWORD or "", + dsn=settings.ADB_DSN or "", + wallet_location=getattr(settings, "ADB_WALLET_LOCATION", None), + wallet_password=getattr(settings, "ADB_WALLET_PASSWORD", None), + table_prefix=(getattr(settings, "ADB_TABLE_PREFIX", "AGENTFW") or "AGENTFW").upper().rstrip("_"), + ) + if not self.cfg.user or not self.cfg.password or not self.cfg.dsn: + raise RuntimeError("ADB_USER, ADB_PASSWORD and ADB_DSN are required") + if auto_init_schema: + self._init_schema() + + @staticmethod + def now() -> datetime: + return datetime.now(timezone.utc) + + def t(self, name: str) -> str: + return f"{self.cfg.table_prefix}_{name}".upper() + + @contextmanager + def connect(self): + import oracledb + oracledb.defaults.fetch_lobs = False + kwargs = {} + if self.cfg.wallet_location: + kwargs["config_dir"] = self.cfg.wallet_location + kwargs["wallet_location"] = self.cfg.wallet_location + if self.cfg.wallet_password: + kwargs["wallet_password"] = self.cfg.wallet_password + conn = oracledb.connect(user=self.cfg.user, password=self.cfg.password, dsn=self.cfg.dsn, **kwargs) + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + async def to_thread(self, func, *args, **kwargs): + return await asyncio.to_thread(func, *args, **kwargs) + + def _exec_ddl_ignore_exists(self, cur, ddl: str): + try: + cur.execute(ddl) + except Exception as exc: + msg = str(exc) + if "ORA-00955" in msg or "ORA-01408" in msg or "ORA-02275" in msg: + return + raise + + def _column_exists(self, cur, table_name: str, column_name: str) -> bool: + cur.execute(""" + select count(*) + from user_tab_columns + where table_name = :table_name + and column_name = :column_name + """, {"table_name": self.t(table_name), "column_name": column_name.upper()}) + return int(cur.fetchone()[0] or 0) > 0 + + def _ensure_column(self, cur, table_name: str, column_name: str, ddl_type: str): + if not self._column_exists(cur, table_name, column_name): + cur.execute(f"alter table {self.t(table_name)} add {column_name.upper()} {ddl_type}") + + def drop_schema(self): + tables = [ + "EVALUATION_RESULT", + "EVALUATION_FINDING", + "EVALUATION_METRIC", + "EVALUATION_ITEM", + "EVALUATION_PROGRESS_EVENT", + "EVALUATION_RUN", + ] + with self.connect() as conn: + cur = conn.cursor() + for table in tables: + try: + cur.execute(f"drop table {self.t(table)} cascade constraints purge") + except Exception as exc: + if "ORA-00942" in str(exc): + continue + raise + + def _init_schema(self): + with self.connect() as conn: + cur = conn.cursor() + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_RUN')} ( + RUN_ID varchar2(64) primary key, + AGENT_ID varchar2(128), + SOURCE varchar2(64), + PERIOD_START timestamp with time zone, + PERIOD_END timestamp with time zone, + STATUS varchar2(32) not null, + TOTAL_ITEMS number default 0 not null, + PROCESSED_ITEMS number default 0 not null, + FAILED_ITEMS number default 0 not null, + RETRY_COUNT number default 0 not null, + ERROR_MESSAGE clob, + LAST_HEARTBEAT_AT timestamp with time zone, + CREATED_AT timestamp with time zone not null, + UPDATED_AT timestamp with time zone not null + ) + """) + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_PROGRESS_EVENT')} ( + ID number generated always as identity primary key, + RUN_ID varchar2(64) not null, + STAGE varchar2(128) not null, + MESSAGE varchar2(1000), + DETAILS_JSON clob check (DETAILS_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_PROGRESS_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_PROGRESS_RUN')} on {self.t('EVALUATION_PROGRESS_EVENT')}(RUN_ID, CREATED_AT)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_ITEM')} ( + ITEM_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + TRACE_ID varchar2(256), + SESSION_ID varchar2(256), + MESSAGE_ID varchar2(256), + AGENT_ID varchar2(128), + CHANNEL varchar2(64), + STATUS varchar2(32) not null, + ATTEMPT_COUNT number default 0 not null, + ERROR_MESSAGE clob, + RAW_JSON clob check (RAW_JSON is json), + CREATED_AT timestamp with time zone not null, + UPDATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_ITEM_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_ITEM_RUN')} on {self.t('EVALUATION_ITEM')}(RUN_ID, STATUS, CREATED_AT)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_RESULT')} ( + RESULT_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + TRACE_ID varchar2(256), + SESSION_ID varchar2(256), + AGENT_ID varchar2(128), + JUDGE_NAME varchar2(128) not null, + JUDGE_TYPE varchar2(32), + SCORE number, + JUDGE_SCORE number, + ACCURACY_SCORE number, + ALUCINATION_SCORE number, + HALLUCINATION_SCORE number, + INFERRED_CSI_SCORE number, + RESOLUTION number, + CONVERSATION_PRECISION number, + TOOL_USAGE_SCORE number, + ROUTING_SCORE number, + RATIONALE clob, + REASONING clob, + RESULT_JSON clob check (RESULT_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_RESULT_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID), + constraint {self.t('FK_EVAL_RESULT_ITEM')} + foreign key (ITEM_ID) references {self.t('EVALUATION_ITEM')}(ITEM_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_RESULT_RUN')} on {self.t('EVALUATION_RESULT')}(RUN_ID, ITEM_ID)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_METRIC')} ( + METRIC_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + METRIC_NAME varchar2(128) not null, + METRIC_VALUE number, + DIMENSIONS_JSON clob check (DIMENSIONS_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_METRIC_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_FINDING')} ( + FINDING_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + SEVERITY varchar2(32), + CATEGORY varchar2(128), + TITLE varchar2(512), + DESCRIPTION clob, + EVIDENCE_JSON clob check (EVIDENCE_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_FINDING_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + + # Non-destructive compatibility for older generated schemas. + for col, typ in [ + ("RETRY_COUNT", "number default 0"), + ("ERROR_MESSAGE", "clob"), + ("LAST_HEARTBEAT_AT", "timestamp with time zone"), + ("UPDATED_AT", "timestamp with time zone"), + ]: + self._ensure_column(cur, "EVALUATION_RUN", col, typ) + for col, typ in [ + ("ID", "number generated always as identity"), + ]: + # Identity column cannot always be added cleanly; ignore if table is old without ID. + try: + self._ensure_column(cur, "EVALUATION_PROGRESS_EVENT", col, typ) + except Exception: + pass + for col, typ in [ + ("JUDGE_NAME", "varchar2(128) default 'unknown_judge' not null"), + ("JUDGE_TYPE", "varchar2(32)"), + ("SCORE", "number"), + ("JUDGE_SCORE", "number"), + ("ACCURACY_SCORE", "number"), + ("ALUCINATION_SCORE", "number"), + ("HALLUCINATION_SCORE", "number"), + ("INFERRED_CSI_SCORE", "number"), + ("RESOLUTION", "number"), + ("CONVERSATION_PRECISION", "number"), + ("TOOL_USAGE_SCORE", "number"), + ("ROUTING_SCORE", "number"), + ("RATIONALE", "clob"), + ("REASONING", "clob"), + ("RESULT_JSON", "clob"), + ]: + self._ensure_column(cur, "EVALUATION_RESULT", col, typ) diff --git a/evaluator/persistence/repository.py b/evaluator/persistence/repository.py new file mode 100644 index 0000000..e89969b --- /dev/null +++ b/evaluator/persistence/repository.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, ItemStatus, RunStatus, TraceJudgeResult, SessionJudgeResult +from evaluator.persistence.oracle_store import OracleStore, _json_dumps, _json_loads + + +class EvaluationRepository: + def __init__(self, auto_init_schema: bool = False): + self.store = OracleStore(settings, auto_init_schema=auto_init_schema) + + def create_run(self, period_start: datetime, period_end: datetime, source: str, agent_id: str | None = None) -> str: + run_id = str(uuid.uuid4()) + now = self.store.now() + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RUN')} + (RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS, TOTAL_ITEMS, + PROCESSED_ITEMS, FAILED_ITEMS, RETRY_COUNT, LAST_HEARTBEAT_AT, CREATED_AT, UPDATED_AT) + values (:run_id, :agent_id, :period_start, :period_end, :source, :status, + 0, 0, 0, 0, :heartbeat_at, :created_at, :updated_at) + """, { + "run_id": run_id, + "agent_id": agent_id, + "period_start": period_start, + "period_end": period_end, + "source": source, + "status": RunStatus.RUNNING.value, + "heartbeat_at": now, + "created_at": now, + "updated_at": now, + }) + return run_id + + async def acreate_run(self, *args, **kwargs): + return await self.store.to_thread(self.create_run, *args, **kwargs) + + def record_progress(self, run_id: str, stage: str, message: str = "", details: dict | None = None): + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_PROGRESS_EVENT')} + (RUN_ID, STAGE, MESSAGE, DETAILS_JSON, CREATED_AT) + values (:run_id, :stage, :message, :details_json, :created_at) + """, { + "run_id": run_id, + "stage": stage, + "message": (message or "")[:1000], + "details_json": _json_dumps(details or {}), + "created_at": self.store.now(), + }) + + async def arecord_progress(self, *args, **kwargs): + return await self.store.to_thread(self.record_progress, *args, **kwargs) + + def insert_items(self, run_id: str, records: list[ConversationRecord]) -> int: + inserted = 0 + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + for record in records: + try: + cur.execute(f""" + insert into {self.store.t('EVALUATION_ITEM')} + (ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL, + STATUS, ATTEMPT_COUNT, RAW_JSON, CREATED_AT, UPDATED_AT) + values (:item_id, :run_id, :trace_id, :session_id, :message_id, :agent_id, + :channel, :status, 0, :raw_json, :created_at, :updated_at) + """, { + "item_id": str(uuid.uuid4()), + "run_id": run_id, + "trace_id": record.trace_id, + "session_id": record.session_id, + "message_id": record.message_id, + "agent_id": record.agent_id, + "channel": record.channel, + "status": ItemStatus.PENDING.value, + "raw_json": record.model_dump_json(), + "created_at": now, + "updated_at": now, + }) + inserted += 1 + except Exception as exc: + if "ORA-00001" not in str(exc): + raise + cur.execute(f""" + update {self.store.t('EVALUATION_RUN')} + set TOTAL_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id + ), + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, {"run_id": run_id, "updated_at": self.store.now()}) + return inserted + + async def ainsert_items(self, *args, **kwargs): + return await self.store.to_thread(self.insert_items, *args, **kwargs) + + def fetch_next_items(self, run_id: str, batch_size: int) -> list[dict]: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL, + STATUS, ATTEMPT_COUNT, RAW_JSON + from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id + and STATUS in (:pending, :failed) + and ATTEMPT_COUNT < :max_attempts + order by CREATED_AT + ) where rownum <= :batch_size + """, { + "run_id": run_id, + "pending": ItemStatus.PENDING.value, + "failed": ItemStatus.FAILED.value, + "max_attempts": settings.max_attempts, + "batch_size": batch_size, + }) + cols = [d[0].lower() for d in cur.description] + return [dict(zip(cols, row)) for row in cur.fetchall()] + + async def afetch_next_items(self, *args, **kwargs): + return await self.store.to_thread(self.fetch_next_items, *args, **kwargs) + + def mark_item_processing(self, item_id: str): + with self.store.connect() as conn: + conn.cursor().execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + ATTEMPT_COUNT = ATTEMPT_COUNT + 1, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.PROCESSING.value, + "updated_at": self.store.now(), + "item_id": item_id, + }) + + async def amark_item_processing(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_processing, *args, **kwargs) + + def mark_item_completed(self, run_id: str, item_id: str): + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.COMPLETED.value, + "updated_at": now, + "item_id": item_id, + }) + self._refresh_run_counters(cur, run_id, now) + + async def amark_item_completed(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_completed, *args, **kwargs) + + def mark_item_failed(self, run_id: str, item_id: str, error: str): + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + ERROR_MESSAGE = :error, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.FAILED.value, + "error": (error or "")[:4000], + "updated_at": now, + "item_id": item_id, + }) + self._refresh_run_counters(cur, run_id, now) + + async def amark_item_failed(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_failed, *args, **kwargs) + + def _refresh_run_counters(self, cur, run_id: str, updated_at): + cur.execute(f""" + update {self.store.t('EVALUATION_RUN')} + set PROCESSED_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id and STATUS = :completed + ), + FAILED_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id and STATUS = :failed + ), + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, { + "run_id": run_id, + "completed": ItemStatus.COMPLETED.value, + "failed": ItemStatus.FAILED.value, + "updated_at": updated_at, + }) + + def save_trace_result(self, run_id: str, item_id: str, record: ConversationRecord, result: TraceJudgeResult): + judge_name = getattr(result, "judge_name", None) or "trace_metrics" + judge_type = (getattr(result, "judge_type", None) or "TRACE").upper() + score = getattr(result, "judgeScore", None) + accuracy = getattr(result, "accuracyScore", None) + alucination = getattr(result, "alucinationScore", None) + rationale = getattr(result, "rationale", None) or "" + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RESULT')} + (RESULT_ID, RUN_ID, ITEM_ID, TRACE_ID, SESSION_ID, AGENT_ID, JUDGE_NAME, + JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE, ALUCINATION_SCORE, + RATIONALE, RESULT_JSON, CREATED_AT) + values (:result_id, :run_id, :item_id, :trace_id, :session_id, :agent_id, + :judge_name, :judge_type, :score, :judge_score, :accuracy_score, + :alucination_score, :rationale, :result_json, :created_at) + """, { + "result_id": str(uuid.uuid4()), + "run_id": run_id, + "item_id": item_id, + "trace_id": record.trace_id, + "session_id": record.session_id, + "agent_id": record.agent_id, + "judge_name": judge_name, + "judge_type": judge_type, + "score": score, + "judge_score": score, + "accuracy_score": accuracy, + "alucination_score": alucination, + "rationale": rationale, + "result_json": result.model_dump_json(), + "created_at": self.store.now(), + }) + + async def asave_trace_result(self, *args, **kwargs): + return await self.store.to_thread(self.save_trace_result, *args, **kwargs) + + def save_session_result(self, run_id: str, session_id: str, agent_id: str | None, result: SessionJudgeResult): + judge_name = getattr(result, "judge_name", None) or "session_metrics" + judge_type = (getattr(result, "judge_type", None) or "SESSION").upper() + rationale = getattr(result, "rationale", None) or "" + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RESULT')} + (RESULT_ID, RUN_ID, SESSION_ID, AGENT_ID, JUDGE_NAME, JUDGE_TYPE, + INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION, RATIONALE, + RESULT_JSON, CREATED_AT) + values (:result_id, :run_id, :session_id, :agent_id, :judge_name, :judge_type, + :csi, :resolution, :precision, :rationale, :result_json, :created_at) + """, { + "result_id": str(uuid.uuid4()), + "run_id": run_id, + "session_id": session_id, + "agent_id": agent_id, + "judge_name": judge_name, + "judge_type": judge_type, + "csi": getattr(result, "inferredCsiScore", None), + "resolution": getattr(result, "resolution", None), + "precision": getattr(result, "conversationPrecision", None), + "rationale": rationale, + "result_json": result.model_dump_json(), + "created_at": self.store.now(), + }) + + async def asave_session_result(self, *args, **kwargs): + return await self.store.to_thread(self.save_session_result, *args, **kwargs) + + def mark_run_status(self, run_id: str, status: RunStatus, error: str | None = None): + with self.store.connect() as conn: + conn.cursor().execute(f""" + update {self.store.t('EVALUATION_RUN')} + set STATUS = :status, + ERROR_MESSAGE = :error, + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, { + "status": status.value, + "error": error, + "updated_at": self.store.now(), + "run_id": run_id, + }) + + async def amark_run_status(self, *args, **kwargs): + return await self.store.to_thread(self.mark_run_status, *args, **kwargs) + + def summarize_run(self, run_id: str) -> dict: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id), + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'COMPLETED'), + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'FAILED'), + (select count(*) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE'), + (select avg(JUDGE_SCORE) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE') + from dual + """, {"run_id": run_id}) + r = cur.fetchone() + return { + "run_id": run_id, + "total_items": int(r[0] or 0), + "completed_items": int(r[1] or 0), + "failed_items": int(r[2] or 0), + "evaluations": int(r[3] or 0), + "avg_score": float(r[4]) if r[4] is not None else None, + } + + async def asummarize_run(self, *args, **kwargs): + return await self.store.to_thread(self.summarize_run, *args, **kwargs) + + def get_run_progress(self, run_id: str, event_limit: int = 20) -> dict: + summary = self.summarize_run(run_id) + total = summary["total_items"] or 0 + done = summary["completed_items"] + summary["failed_items"] + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select STAGE, MESSAGE, DETAILS_JSON, CREATED_AT + from {self.store.t('EVALUATION_PROGRESS_EVENT')} + where RUN_ID = :run_id + order by CREATED_AT desc + ) where rownum <= :max_rows + """, {"run_id": run_id, "max_rows": event_limit}) + events = [ + { + "stage": s, + "message": m, + "details": _json_loads(d.read() if hasattr(d, "read") else d, {}), + "created_at": str(c), + } + for s, m, d, c in cur.fetchall() + ] + return { + **summary, + "done_items": done, + "percent_complete": round((done / total) * 100, 2) if total else 0.0, + "events": events, + } + + async def aget_run_progress(self, *args, **kwargs): + return await self.store.to_thread(self.get_run_progress, *args, **kwargs) + + def list_runs(self, limit: int = 20): + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS, + TOTAL_ITEMS, PROCESSED_ITEMS, FAILED_ITEMS, CREATED_AT, UPDATED_AT + from {self.store.t('EVALUATION_RUN')} + order by CREATED_AT desc + ) where rownum <= :max_rows + """, {"max_rows": limit}) + return [ + { + "run_id": r[0], + "agent_id": r[1], + "period_start": str(r[2]), + "period_end": str(r[3]), + "source": r[4], + "status": r[5], + "total_items": int(r[6] or 0), + "processed_items": int(r[7] or 0), + "failed_items": int(r[8] or 0), + "created_at": str(r[9]), + "updated_at": str(r[10]), + } + for r in cur.fetchall() + ] + + async def alist_runs(self, *args, **kwargs): + return await self.store.to_thread(self.list_runs, *args, **kwargs) + + def list_results(self, run_id: str, limit: int = 100) -> list[dict]: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select JUDGE_NAME, JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE, + ALUCINATION_SCORE, INFERRED_CSI_SCORE, RESOLUTION, + CONVERSATION_PRECISION, RATIONALE, TRACE_ID, SESSION_ID, CREATED_AT + from {self.store.t('EVALUATION_RESULT')} + where RUN_ID = :run_id + order by CREATED_AT desc + """, {"run_id": run_id}) + return [ + { + "judge_name": r[0], + "judge_type": r[1], + "score": r[2], + "judge_score": r[3], + "accuracy_score": r[4], + "alucination_score": r[5], + "inferred_csi_score": r[6], + "resolution": r[7], + "conversation_precision": r[8], + "rationale": r[9], + "trace_id": r[10], + "session_id": r[11], + "created_at": str(r[12]), + } + for r in cur.fetchall()[:limit] + ] + + async def alist_results(self, *args, **kwargs): + return await self.store.to_thread(self.list_results, *args, **kwargs) diff --git a/evaluator/prompts/__pycache__/loader.cpython-313.pyc b/evaluator/prompts/__pycache__/loader.cpython-313.pyc new file mode 100644 index 0000000..d3a7bd0 Binary files /dev/null and b/evaluator/prompts/__pycache__/loader.cpython-313.pyc differ diff --git a/evaluator/prompts/loader.py b/evaluator/prompts/loader.py new file mode 100644 index 0000000..5e800f7 --- /dev/null +++ b/evaluator/prompts/loader.py @@ -0,0 +1,11 @@ +from __future__ import annotations +import yaml +from evaluator.config.settings import settings + + +def load_prompt(path: str, key: str) -> str: + p = settings.path(path) + data = yaml.safe_load(p.read_text()) or {} + if key not in data: + raise KeyError(f"Prompt key {key!r} not found in {p}") + return str(data[key]) diff --git a/evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc b/evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc new file mode 100644 index 0000000..0238fa1 Binary files /dev/null and b/evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc differ diff --git a/evaluator/publishers/langfuse_scores.py b/evaluator/publishers/langfuse_scores.py new file mode 100644 index 0000000..be33e90 --- /dev/null +++ b/evaluator/publishers/langfuse_scores.py @@ -0,0 +1,22 @@ +from __future__ import annotations +import httpx +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, TraceJudgeResult + +class LangfuseScorePublisher: + async def publish_trace_score(self, record: ConversationRecord, result: TraceJudgeResult): + if not settings.can_publish_langfuse_scores or not record.trace_id: + return None + auth = (settings.langfuse_public_key, settings.langfuse_secret_key) + payloads = [ + {'traceId': record.trace_id, 'name': 'offline_judge_score', 'value': result.judgeScore, 'comment': result.rationale}, + {'traceId': record.trace_id, 'name': 'offline_accuracy_score', 'value': result.accuracyScore, 'comment': result.rationale}, + {'traceId': record.trace_id, 'name': 'offline_alucination_score', 'value': result.alucinationScore, 'comment': result.rationale}, + ] + async with httpx.AsyncClient(base_url=settings.langfuse_host, timeout=30) as client: + for payload in payloads: + resp = await client.post('/api/public/scores', json=payload, auth=auth) + if resp.status_code >= 400: + # Don't fail the run because score publishing is supplementary. + return {'ok': False, 'status': resp.status_code, 'body': resp.text} + return {'ok': True} diff --git a/k8s/cronjob.yaml b/k8s/cronjob.yaml new file mode 100644 index 0000000..c317248 --- /dev/null +++ b/k8s/cronjob.yaml @@ -0,0 +1,20 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: agent-framework-evaluator +spec: + schedule: "0 2 * * *" + suspend: true + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: evaluator + image: agent-framework-evaluator:latest + command: ["python", "-m", "evaluator.cli", "run-agents", "--source", "langfuse"] + envFrom: + - secretRef: + name: agent-framework-evaluator-env diff --git a/output/.DS_Store b/output/.DS_Store new file mode 100644 index 0000000..5008ddf Binary files /dev/null and b/output/.DS_Store differ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d3968f7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[project] +name = "agent-framework-evaluator" +version = "0.2.0" +description = "Offline LLM-as-a-Judge evaluator for Agent Framework conversations" +requires-python = ">=3.11" +dependencies = [ + "python-dotenv>=1.0.1", + "pydantic>=2.7.0", + "pydantic-settings>=2.2.1", + "oracledb>=2.4.0", + "httpx>=0.27.0", + "pyyaml>=6.0.1", + "typer>=0.12.3", + "click>=8.1.7", + "rich>=13.7.0", + "fastapi>=0.111.0", + "uvicorn>=0.30.0" +] + +[project.scripts] +af-evaluator = "evaluator.cli:app" + +[tool.setuptools.packages.find] +where = ["."] +include = ["evaluator*"]