commit b2a368d71ab6c2367161e31509b7ca889ecf49d4 Author: Cristiano Hoshikawa Date: Wed Sep 9 16:50:00 2026 -0300 first commit diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..5cf0f44 Binary files /dev/null and b/.DS_Store differ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..cca14aa --- /dev/null +++ b/.env.example @@ -0,0 +1,61 @@ +# Oracle Autonomous Database / Wallet - same pattern used by Agent Framework +APP_ENV=dev +ADB_USER=admin +ADB_PASSWORD=change-me +ADB_DSN=oradb23ai_high +ADB_WALLET_LOCATION=/path/to/Wallet_ORADB23ai +ADB_WALLET_PASSWORD=change-me +ADB_TABLE_PREFIX=AGENTFW + +# Langfuse manual/global fallback (generic `run` only). +# Productive `run-agents` uses LANGFUSE_CREDENTIALS_CONFIG_PATH below. +ENABLE_LANGFUSE=true +LANGFUSE_PUBLIC_KEY=pk-lf-... +LANGFUSE_SECRET_KEY=sk-lf-... +LANGFUSE_HOST=http://localhost:3005 +PUBLISH_LANGFUSE_SCORES=false + +# LLM - Agent Framework compatible providers: mock, oci_openai, oci_sdk, openai_compatible +LLM_PROVIDER=oci_sdk +LLM_PROFILE=judge +LLM_PROFILES_PATH=configs/llm_profiles/llm_profiles.yaml +LLM_TEMPERATURE=0 +LLM_MAX_TOKENS=900 +LLM_TIMEOUT_SECONDS=120 + +# OCI SDK (native OCI GenAI; dedicated endpoint supported) +OCI_AUTH_MODE=config_file +OCI_CONFIG_FILE=~/.oci/config +OCI_PROFILE=DEFAULT +OCI_REGION=sa-saopaulo-1 +OCI_COMPARTMENT_ID= +OCI_GENAI_BASE_URL=https://inference.generativeai.sa-saopaulo-1.oci.oraclecloud.com +OCI_GENAI_MODEL=meta.llama-3.3-70b-instruct +# For dedicated endpoints, set this to ocid1.generativeaiendpoint...; +# alternatively OCI_GENAI_MODEL may itself contain the endpoint OCID. +OCI_GENAI_ENDPOINT_ID= +OCI_GENAI_PROJECT_OCID= + +# OCI OpenAI-compatible only (when LLM_PROVIDER=oci_openai) +OCI_GENAI_API_KEY= + +# Execution +AGENTS_CONFIG_PATH=configs/judge/agents.yaml +TRACE_PROMPT_PATH=configs/judge/trace_metrics.yaml +SESSION_PROMPT_PATH=configs/judge/session_metrics.yaml +OUTPUT_DIR=output +BATCH_SIZE=50 +MAX_ATTEMPTS=3 +CONTEXT_TURN_LIMIT=6 + +# GCS export required by TIM specification +ENABLE_GCS_UPLOAD=true +# Optional override. If empty, dev/fqa/prd buckets from the TIM specification are used. +JUDGE_GCS_BUCKET= +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +GOOGLE_APPLICATION_CREDENTIALS=configs/GCP_ACCESS_KEY.json + +# Langfuse credentials are declared in one external YAML file. +# In production, mount it from a Kubernetes Secret. +LANGFUSE_CREDENTIALS_CONFIG_PATH=configs/langfuse/langfuse_credentials.yaml + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0c7addf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +WORKDIR /app +COPY . /app +RUN pip install --no-cache-dir -e . +CMD ["python", "-m", "evaluator.cli", "run-agents", "--source", "langfuse"] diff --git a/Documentation/llm-as-a-judge.pdf b/Documentation/llm-as-a-judge.pdf new file mode 100644 index 0000000..f8db66f Binary files /dev/null and b/Documentation/llm-as-a-judge.pdf differ diff --git a/README.en-US.md b/README.en-US.md new file mode 100644 index 0000000..00511a4 --- /dev/null +++ b/README.en-US.md @@ -0,0 +1,1536 @@ +# Agent Framework OCI Evaluator — LLM-as-a-Judge + +**Language:** English | [Português](README.md) + +## 1. Overview + +The **Agent Framework OCI Evaluator** is the batch component responsible for asynchronously evaluating the quality of conversations produced by agents built on the Agent Framework OCI. + +The service reads sessions and traces from **Langfuse v3**, deterministically samples sessions, calculates quality metrics with an **LLM-as-a-Judge**, persists state and results in **Oracle Autonomous Database**, generates the official `.TXT.GZ` artifact, publishes that artifact to **Google Cloud Storage (GCS)**, and audits the evaluator job itself in Langfuse. + +It is designed for **D+1 execution** as a **Kubernetes CronJob**, with no runtime HTTP/gRPC endpoint. + +> **Recommended deployment model:** one evaluator instance per agent/squad. The code is generic and supports multiple agents in configuration, but in production each deployment should normally enable only the agent owned by that squad. This keeps credentials, ownership, scheduling, and troubleshooting isolated. + +--- + +## 2. Why this component exists + +Running generative-quality evaluation synchronously inside the agent request path would increase latency, cost, and coupling. The evaluator moves that responsibility out of the online interaction path. + +The architecture is: + +```text +Online interaction + | + v +Agent Framework + | + +--------------------> Langfuse v3 + | + | traces / sessions + v + D+1 Evaluator (CronJob) + | + +------------------+------------------+ + | | | + v v v + OCI GenAI ADB GCS + LLM-as-a-Judge state/results official .TXT.GZ + | + +-----------------------> Langfuse + audit / optional scores +``` + +This separation provides: + +- no evaluator latency in the online interaction; +- replay and auditability; +- configurable sampling per agent; +- isolation of evaluation failures from the online agent; +- support for dedicated OCI GenAI models; +- an official artifact for the TIM analytics layer; +- no code coupling to specific agent names, buckets, or credentials. + +--- + +## 3. Core concepts + +### 3.1. Trace + +A **trace** represents an interaction evaluated individually. The evaluator calculates these trace-level metrics: + +- `judgeScore` — overall response quality; +- `accuracyScore` — factual/contextual accuracy; +- `alucinationScore` — degree of unsupported claims; +- `promptLength` — raw character count of `trace.input`. + +### 3.2. Session + +A **session** represents the complete conversation. A session normally contains multiple traces. + +Session-level metrics are calculated once and replicated to every trace row in that session: + +- `inferredCsiScore`; +- `resolution`; +- `conversationPrecision`; +- `loop`. + +### 3.3. Session-based sampling + +Sampling is performed on **sessions**, never on isolated traces. + +The deterministic seed is: + +```text +agentId:reference_date +``` + +Example: + +```text +billing:2026-08-13 +``` + +Two executions for the same agent, date, and percentage therefore select the same sessions. + +Once a session is selected, **all traces in that session** are retained. + +### 3.4. BRT time window + +`run-agents` calculates the reference date using: + +```text +D - daysBack +``` + +with explicit timezone: + +```text +America/Sao_Paulo +``` + +With `daysBack: 1`, a run on August 14 evaluates the full August 13 window, from 00:00:00 through 23:59:59.999999 BRT. + +### 3.5. CONTEXT_TURN_LIMIT + +`CONTEXT_TURN_LIMIT` controls how many previous conversation turns are included in trace-level judge context. + +Default: + +```env +CONTEXT_TURN_LIMIT=6 +``` + +The variable exists because a response should not be judged outside the context of its conversation. Increasing the value provides more context, but also increases tokens, cost, and judge latency. + +### 3.6. Loop + +The `loop` metric is not simple text repetition. It represents the maximum consecutive **`metadata.intent` streak** in a session. + +Example: + +```text +billing +billing +billing +payment +payment +``` + +Result: + +```text +loop = 3 +``` + +--- + +## 4. End-to-end execution flow + +The production flow is executed with: + +```bash +python -m evaluator.cli run-agents +``` + +or through the installed entrypoint: + +```bash +af-evaluator run-agents +``` + +Steps: + +```text +1. Load agents.yaml +2. Compute D-daysBack in BRT +3. Resolve Langfuse credentials for the agent +4. Read ALL native sessions in the period, paginated +5. Read ALL traces in the period, paginated +6. Read observations in bulk/paginated mode +7. Recover the real session_id from metadata when a legacy trace has no native sessionId +8. Filter agent traces/aliases +9. Deterministically sample logical sessions +10. Keep all traces belonging to sampled sessions +11. Persist items in ADB +12. Run trace-level judge +13. Run session-level judge +14. Calculate local loop metric +15. Generate .TXT.GZ +16. Upload to GCS +17. Delete local file after successful upload +18. Update judge-job-{agentId} audit trace in Langfuse +``` + +### 4.1. Why collection avoids N+1 + +The implementation avoids the old pattern: + +```text +1 trace + -> GET trace detail + -> GET observations +``` + +repeated once per trace. + +Collection now uses bulk pagination: + +```text +/api/public/sessions page 1..N +/api/public/traces page 1..N +/api/public/observations page 1..N +``` + +Therefore 10,000 traces do not produce approximately 20,000 additional requests. Request count grows with the number of API pages instead. + +The current `PAGE_SIZE` is 100. + +### 4.2. Compatibility with traces without native `sessionId` + +Some historical instrumentations store the session in trace/observation metadata without promoting it to Langfuse's native `trace.sessionId`. The evaluator recovers the real session using this precedence: + +```text +trace.sessionId / trace.session_id + -> trace.metadata.session_id / sessionId / conversation_key + -> observation.metadata.session_id / sessionId / conversation_key + -> business_context.session_key (last metadata fallback) +``` + +`trace.id` is **never** used as a `session_id`. This preserves session sampling and session-level metrics for historical traces without creating an artificial one-session-per-trace model. Collection statistics expose `native_sessions_read`, `metadata_sessions_recovered`, and `traces_without_session`. + +--- + +## 5. Component architecture + +```text +agent_framework_oci_evaluator/ +├── evaluator/ +│ ├── analytics/ +│ │ └── vloop.py +│ ├── collectors/ +│ │ ├── langfuse.py +│ │ ├── langfuse_credentials.py +│ │ ├── agent_framework.py +│ │ └── mock.py +│ ├── config/ +│ │ ├── agents.py +│ │ └── settings.py +│ ├── core/ +│ │ └── models.py +│ ├── identity/ +│ │ └── resolver.py +│ ├── judges/ +│ │ └── llm_judge.py +│ ├── llm/ +│ │ ├── client.py +│ │ └── profile_resolver.py +│ ├── output/ +│ │ └── legacy_exporter.py +│ ├── persistence/ +│ │ ├── oracle_store.py +│ │ └── repository.py +│ ├── prompts/ +│ │ └── loader.py +│ ├── publishers/ +│ │ ├── gcs_export.py +│ │ ├── langfuse_job_audit.py +│ │ └── langfuse_scores.py +│ ├── cli.py +│ ├── engine.py +│ └── time_utils.py +├── configs/ +│ ├── identity.yaml +│ ├── judge/ +│ │ ├── agents.yaml +│ │ ├── storage.yaml +│ │ ├── trace_metrics.yaml +│ │ └── session_metrics.yaml +│ ├── langfuse/ +│ │ └── langfuse_credentials.example.yaml +│ └── llm_profiles/ +│ └── llm_profiles.yaml +├── deploy/ +├── tests/ +├── .env.example +├── Dockerfile +└── pyproject.toml +``` + +### 5.1. Collectors + +Collectors transform external data sources into `ConversationRecord` objects. + +Available sources: + +```text +langfuse +agent_framework +mock +``` + +`langfuse` is the default production source. + +### 5.2. EvaluationEngine + +`evaluator/engine.py` orchestrates: + +- run creation; +- collection; +- sampling; +- persistence; +- judging; +- export; +- upload; +- audit. + +### 5.3. Persistence + +Execution state and results are persisted in Oracle Database. The schema is created automatically by `init-db`. + +Main tables: + +```text +EVALUATION_RUN +EVALUATION_PROGRESS_EVENT +EVALUATION_ITEM +EVALUATION_RESULT +EVALUATION_METRIC +EVALUATION_FINDING +``` + +The table prefix is configured through: + +```env +ADB_TABLE_PREFIX=AGENTFW +``` + +### 5.4. Publishers + +- `gcs_export.py` — uploads the official artifact; +- `langfuse_job_audit.py` — writes the `judge-job-{agentId}` audit trace; +- `langfuse_scores.py` — optionally publishes trace scores back to Langfuse. + +> **Note:** production job audit uses the agent's dynamic credentials. Optional score publishing (`PUBLISH_LANGFUSE_SCORES=true`) still uses global `LANGFUSE_HOST/PUBLIC_KEY/SECRET_KEY`; keep it disabled unless an appropriate global project is configured. + +--- + +## 6. Agent configuration + +Default file: + +```text +configs/judge/agents.yaml +``` + +Example: + +```yaml +agents: + - agentId: billing + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: + - telecom_contas + gcs_prefix: agnt_ai_contas +``` + +Fields: + +| Field | Description | +|---|---| +| `agentId` | official logical agent identity | +| `enabled` | enables/disables the agent in that deployment | +| `daysBack` | number of days to subtract from the current BRT date | +| `percentage` | percentage of sessions evaluated, from 0.0 to 1.0 | +| `langfuse_agent_aliases` | technical agent names accepted in traces; when populated, it is an explicit filter regardless of the Langfuse credential source | +| `gcs_prefix` | agent prefix inside the bucket | + +`agent_id` and `days_back` are also accepted for compatibility, but `agentId` / `daysBack` is the recommended form. + +**`langfuse_agent_aliases` semantics:** when the list contains values, only traces whose agent identifier matches one of the aliases are eligible for sampling and evaluation. When the list is empty, the evaluator does not apply an agent-identity filter and treats the configured Langfuse project/endpoint as exclusive to that entry. + +### 6.1. One evaluator per agent + +For the recommended one-instance-per-agent deployment model, keep only one agent enabled in that deployment: + +```yaml +agents: + - agentId: billing + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: + - telecom_contas + - billing_agent + gcs_prefix: agnt_ai_contas +``` + +The same evaluator code can be reused by another repository/deployment simply by changing configuration files. + +--- + +## 7. Dynamic Langfuse credentials + +There are no hardcoded variables such as: + +```text +LANGFUSE_BILLING_* +LANGFUSE_SALES_* +``` + +The evaluator reads one declarative definition: + +```env +LANGFUSE_CREDENTIALS_CONFIG_PATH=configs/langfuse/langfuse_credentials.yaml +``` + +Create it from the example: + +```bash +cp configs/langfuse/langfuse_credentials.example.yaml \ + configs/langfuse/langfuse_credentials.yaml +``` + +Example: + +```yaml +langfuse: + agents: + - agentId: billing + host: https://langfuse.example.internal + publicKey: pk-... + secretKey: sk-... + + - agentId: another-agent + host: https://another-langfuse.internal + publicKey: pk-... + secretKey: sk-... +``` + +The resolver iterates through all entries. Adding a new agent requires no Python change. + +### 7.1. Environment/file references + +To avoid storing secrets in Git: + +```yaml +langfuse: + agents: + - agentId: billing + hostEnv: BILLING_LF_HOST + publicKeyEnv: BILLING_LF_PUBLIC + secretKeyFile: /var/secrets/langfuse/billing-secret-key +``` + +Supported fields: + +```text +host / hostEnv / hostFile +publicKey / publicKeyEnv / publicKeyFile +secretKey / secretKeyEnv / secretKeyFile +``` + +Resolution order for each field: + +```text +direct value -> declared environment variable -> declared file +``` + +### 7.2. Global credentials + +These variables remain available only for the generic manual `run` command: + +```env +LANGFUSE_HOST=http://localhost:3005 +LANGFUSE_PUBLIC_KEY=... +LANGFUSE_SECRET_KEY=... +``` + +The production `run-agents` flow resolves credentials from the per-agent declarative file. + +--- + +## 8. LLM configuration + +Supported providers: + +```text +mock +oci_openai +oci_sdk +openai_compatible +``` + +The recommended provider for dedicated OCI models is: + +```env +LLM_PROVIDER=oci_sdk +``` + +### 8.1. OCI SDK + +Local example using `~/.oci/config`: + +```env +LLM_PROVIDER=oci_sdk +OCI_AUTH_MODE=config_file +OCI_CONFIG_FILE=~/.oci/config +OCI_PROFILE=DEFAULT +OCI_REGION=sa-saopaulo-1 +OCI_COMPARTMENT_ID=ocid1.compartment... +OCI_GENAI_BASE_URL=https://inference.generativeai.sa-saopaulo-1.oci.oraclecloud.com +OCI_GENAI_MODEL=meta.llama-3.3-70b-instruct +``` + +Authentication modes supported by the code: + +```text +config_file +instance_principal +resource_principal +oke_workload_identity +workload_identity +``` + +### 8.2. Dedicated endpoint + +For a dedicated endpoint: + +```env +OCI_GENAI_BASE_URL=https://.inference.generativeai..oci.oraclecloud.com +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... +``` + +The dedicated endpoint OCID may also be provided in `OCI_GENAI_MODEL`. + +The provider selects `DedicatedServingMode` when an endpoint OCID is available; otherwise it uses `OnDemandServingMode`. + +### 8.3. Judge profiles + +File: + +```text +configs/llm_profiles/llm_profiles.yaml +``` + +Example: + +```yaml +profiles: + judge: + temperature: 0 + max_tokens: 900 +``` + +Selected through: + +```env +LLM_PROFILE=judge +``` + +--- + +## 9. Prompts and metrics + +### 9.1. Trace metrics + +File: + +```text +configs/judge/trace_metrics.yaml +``` + +The judge must return JSON such as: + +```json +{ + "judgeScore": 0.0, + "accuracyScore": 0.0, + "alucinationScore": 0.0, + "rationale": "..." +} +``` + +If parsing fails: + +```text +1st attempt + -> parse fails +2nd attempt with retry hint + -> parse fails again +trace-level metrics = None + -> exported as "0.00" +``` + +### 9.2. Session metrics + +File: + +```text +configs/judge/session_metrics.yaml +``` + +Expected result: + +```json +{ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "..." +} +``` + +Session-level values are replicated to every trace row belonging to that session. + +--- + +## 10. Identity normalization + +File: + +```text +configs/identity.yaml +``` + +The identity layer converts channel/domain-specific identifiers into canonical keys such as: + +```text +customer_key +contract_key +interaction_key +account_key +resource_key +session_key +``` + +This prevents `cpf`, `msisdn`, `invoice_id`, `order_id`, `ura_call_id`, and similar names from being spread throughout evaluator code. + +Source ordering works as fallback priority. + +Example: + +```yaml +session_key: + sources: + - business_context.session_key + - session_key + - conversation_key + - session_id +``` + +--- + +## 11. GCS configuration + +Buckets live outside the code in: + +```text +configs/judge/storage.yaml +``` + +Example: + +```yaml +storage: + buckets: + dev: tim-bigdata-dev-ca1f-raw-standard-agnt-ai-atendimento + fqa: tim-bigdata-fqa-60ce-raw-standard-agnt-ai-atendimento + prd: tim-bigdata-prod-e305-raw-standard-agnt-ai-atendimento +``` + +Selection: + +```env +APP_ENV=dev +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +``` + +Operational override: + +```env +JUDGE_GCS_BUCKET=my-bucket +``` + +Enable the production flow with: + +```env +ENABLE_GCS_UPLOAD=true +GOOGLE_APPLICATION_CREDENTIALS=/path/GCP_ACCESS_KEY.json +``` + +`run-agents` treats GCS as mandatory. If `ENABLE_GCS_UPLOAD=false` or upload fails, the agent run finishes as `FAILED`. + +--- + +## 12. Output file + +Pattern: + +```text +AGENTE_{agentId}_LLM_JUDGE_{YYYYMMDD}_{HHMMSS}.TXT.GZ +``` + +Example: + +```text +AGENTE_billing_LLM_JUDGE_20260814_020015.TXT.GZ +``` + +Header: + +```text +"judgeScore"|;"accuracyScore"|;"alucinationScore"|;"promptLength"|;"loop"|;"inferredCsiScore"|;"resolution"|;"conversationPrecision"|;"uraCallId"|;"channelId"|;"sessionId"|;"messageId" +``` + +Final line: + +```text +"TOTAL"|;"" +``` + +After a successful GCS upload, the local file is deleted. + +--- + +## 13. Local installation + +### 13.1. Prerequisites + +- Python 3.11+; +- access to Oracle Database/Autonomous Database; +- Oracle Wallet when required; +- access to Langfuse v3; +- OCI Generative AI access for real judge execution; +- GCP credentials for production upload. + +### 13.2. Create the virtual environment + +Linux/WSL/macOS: + +```bash +python -m venv .venv +source .venv/bin/activate +``` + +Windows PowerShell: + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +``` + +### 13.3. Install + +Recommended development mode: + +```bash +python -m pip install --upgrade pip setuptools wheel +pip install -e . +``` + +Validate: + +```bash +pip check +python -c "import oci, httpx, oracledb; import google.cloud.storage; print('Dependencies OK')" +``` + +--- + +## 14. Local configuration step by step + +### Step 1 — `.env` + +```bash +cp .env.example .env +``` + +Edit the real values. + +Typical minimum configuration: + +```env +APP_ENV=dev + +ADB_USER=admin +ADB_PASSWORD=... +ADB_DSN=... +ADB_WALLET_LOCATION=/path/to/wallet +ADB_WALLET_PASSWORD=... +ADB_TABLE_PREFIX=AGENTFW + +ENABLE_LANGFUSE=true +LANGFUSE_CREDENTIALS_CONFIG_PATH=configs/langfuse/langfuse_credentials.yaml + +LLM_PROVIDER=oci_sdk +LLM_PROFILE=judge +OCI_AUTH_MODE=config_file +OCI_CONFIG_FILE=~/.oci/config +OCI_PROFILE=DEFAULT +OCI_REGION=sa-saopaulo-1 +OCI_COMPARTMENT_ID=... +OCI_GENAI_BASE_URL=https://... +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... + +CONTEXT_TURN_LIMIT=6 +BATCH_SIZE=50 +MAX_ATTEMPTS=3 + +ENABLE_GCS_UPLOAD=true +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +GOOGLE_APPLICATION_CREDENTIALS=/path/GCP_ACCESS_KEY.json +``` + +### Step 2 — Langfuse + +```bash +cp configs/langfuse/langfuse_credentials.example.yaml \ + configs/langfuse/langfuse_credentials.yaml +``` + +Declare the agent that will be executed. + +### Step 3 — agents.yaml + +Adjust: + +```text +configs/judge/agents.yaml +``` + +For a dedicated instance, keep only that squad's agent enabled. + +### Step 4 — validate configuration + +```bash +python -m evaluator.cli show-config +``` + +### Step 5 — initialize the database + +```bash +python -m evaluator.cli init-db +``` + +> `reset-db` drops and recreates the evaluator schema. Use it only in a controlled environment. + +--- + +## 15. CLI + +The actual help for the current version is: + +```bash +python -m evaluator.cli --help +``` + +Available commands: + +```text +init-db +reset-db +show-config +run +run-agents +progress +runs +``` + +### 15.1. `run` — manual/diagnostic execution + +Syntax: + +```bash +python -m evaluator.cli run PERIOD_START PERIOD_END [OPTIONS] +``` + +Example: + +```bash +python -m evaluator.cli run \ + 2026-08-12T00:00:00 \ + 2026-08-12T23:59:59 \ + --source langfuse \ + --limit 10 +``` + +Options: + +```text +--source TEXT default: langfuse +--limit INTEGER +--show-progress / --no-show-progress +``` + +**Important:** `run` is a generic manual path. It collects, evaluates, and persists results, but does not execute the full production per-agent contract (configured sampling, GCS export, and job audit). Use `run-agents` for certification/production behavior. + +### 15.2. `run-agents` — production flow + +```bash +python -m evaluator.cli run-agents +``` + +One agent: + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --source langfuse +``` + +Limited test: + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --limit 10 +``` + +Options: + +```text +--source TEXT default: langfuse +--agent-id TEXT +--limit INTEGER +``` + +`run-agents` uses the agent's `daysBack`, `percentage`, credentials, and `gcs_prefix`. + +### 15.3. List runs + +```bash +python -m evaluator.cli runs +``` + +or: + +```bash +python -m evaluator.cli runs --limit 50 +``` + +### 15.4. Inspect run progress + +```bash +python -m evaluator.cli progress +``` + +With more events: + +```bash +python -m evaluator.cli progress --events 100 +``` + +Typical production events: + +```text +LANGFUSE_JOB_AUDIT +RUN_CREATED +COLLECTING +COLLECTED +SAMPLED +ITEMS_INSERTED +BATCH_STARTED +ITEM_COMPLETED +SESSION_JUDGE_COMPLETED +COMPLETED +EXPORTED +GCS_UPLOADED +LOCAL_FILE_DELETED +LANGFUSE_JOB_AUDIT +``` + +--- + +## 16. Tests + +Install `pytest` if needed: + +```bash +pip install pytest +``` + +### 16.1. All tests + +```bash +python -m pytest -v +``` + +### 16.2. TIM specification tests + +```bash +python -m pytest tests/test_tim_spec.py -v +``` + +### 16.3. Extended compliance tests + +```bash +python -m pytest tests/test_full_tim_compliance.py -v +``` + +### 16.4. Hardcode regression tests + +```bash +python -m pytest tests/test_no_product_hardcodes.py -v +``` + +### 16.5. Recommended release suite + +```bash +python -m pytest \ + tests/test_tim_spec.py \ + tests/test_full_tim_compliance.py \ + tests/test_no_product_hardcodes.py \ + -v +``` + +Validate syntax/imports: + +```bash +python -m compileall evaluator +``` + +--- + +## 17. Recommended local functional test + +### 17.1. Mock smoke test + +Configure: + +```env +LLM_PROVIDER=mock +``` + +Then run a small period using the mock source when desired: + +```bash +python -m evaluator.cli run \ + 2026-08-12T00:00:00 \ + 2026-08-12T23:59:59 \ + --source mock +``` + +### 17.2. Small Langfuse test + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --limit 5 +``` + +Then: + +```bash +python -m evaluator.cli runs --limit 5 +python -m evaluator.cli progress --events 100 +``` + +Confirm: + +- `COLLECTED` > 0; +- `SAMPLED` lists selected sessions; +- items finish without `ITEM_FAILED`; +- `EXPORTED` appears; +- `GCS_UPLOADED` contains a `gs://...` URI; +- `LOCAL_FILE_DELETED` appears; +- `completed` audit was confirmed. + +--- + +## 18. Docker + +Local build: + +```bash +docker build -t agent-framework-evaluator:local . +``` + +The production runtime is batch-only. The container does not need to expose an HTTP port. + +In Kubernetes, the `CronJob` runs: + +```text +python -m evaluator.cli run-agents --source langfuse +``` + +--- + +## 19. Kubernetes / OKE + +The **evaluator itself remains CronJob-only**, as required by the specification: it does not open an HTTP/gRPC port and evaluation processing continues to run exclusively in `evaluator-batch`. + +The web interface is provided by a **separate read-only component**, `evaluator-monitor`. It does not execute evaluations, mutate runs, or change the evaluator batch contract. It only reads run/progress/result tables from Autonomous Database. + +```text +Evaluator batch (evaluator-batch CronJob) + │ + ├── writes EVALUATION_RUN + ├── writes EVALUATION_PROGRESS_EVENT + └── writes EVALUATION_RESULT + │ + ▼ + Autonomous Database + ▲ + │ read-only + Evaluator Monitor (HTTP Deployment) + │ + └── /ui + monitoring APIs +``` + +Kustomize resources: + +```text +Namespace +ServiceAccount +ConfigMap +CronJob evaluator-batch +Deployment evaluator-monitor +Service evaluator-monitor (ClusterIP) +``` + +No Ingress is created by default, so the UI is not automatically exposed outside the cluster. + +Current CronJob: + +```yaml +schedule: "0 2 * * *" +timeZone: "America/Sao_Paulo" +suspend: false +concurrencyPolicy: Forbid +``` + +### 19.1. Prepare deployment files + +```bash +cp deploy/env/evaluator.env.example deploy/env/evaluator.env +cp deploy/langfuse/langfuse_credentials.example.yaml \ + deploy/langfuse/langfuse_credentials.yaml +``` + +Place the Oracle wallet in: + +```text +deploy/wallet/ +``` + +GCP credential: + +```text +deploy/gcp/GCP_ACCESS_KEY.json +``` + +### 19.2. Create secrets + +DEV: + +```bash +./deploy/scripts/create-secrets.sh dev +``` + +HML: + +```bash +./deploy/scripts/create-secrets.sh hml +``` + +PROD: + +```bash +./deploy/scripts/create-secrets.sh prod +``` + +Created/updated secrets: + +```text +evaluator-secrets +evaluator-wallet +evaluator-langfuse +evaluator-gcp (when the GCP file exists) +``` + +### 19.3. Build and push + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG= +./deploy/scripts/build-push.sh prod +``` + +### 19.4. Deploy + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG= +./deploy/scripts/deploy.sh prod +``` + +By default the script also runs the database initialization job. + +To disable it: + +```bash +export RUN_INIT_DB=false +``` + +### 19.5. Status + +```bash +./deploy/scripts/status.sh prod +``` + +or directly: + +```bash +kubectl -n agent-evaluator-prod get cronjob,jobs,pods +``` + +### 19.6. Immediate execution outside the schedule + +```bash +./deploy/scripts/run-now.sh prod +``` + +The script creates a Job from the CronJob. + +Follow logs: + +```bash +kubectl -n agent-evaluator-prod get jobs,pods +kubectl -n agent-evaluator-prod logs -f job/ +``` + + +### 19.7. Web monitoring interface + +The UI is served by the independent `evaluator-monitor` component: + +```text +GET /ui +GET /health +GET /api/runs +GET /api/runs/{run_id} +GET /api/runs/{run_id}/results +``` + +To access a Kubernetes deployment locally without creating an Ingress: + +```bash +kubectl -n agent-evaluator-prod port-forward svc/evaluator-monitor 8080:8080 +``` + +Then open: + +```text +http://localhost:8080/ui +``` + +The page shows recent runs, `RUN_ID`, agent, status, progress, events, and results. The browser auto-refreshes every 5 seconds. + +To run the monitor locally: + +```bash +python -m uvicorn evaluator_monitor.app:app --host 0.0.0.0 --port 8080 +``` + +The monitor uses the same `ADB_*`/wallet settings as the evaluator. Functionally it is **read-only**: there is no endpoint to start, cancel, delete, or modify runs. + +> Security recommendation: keep the Service as `ClusterIP`. If external exposure is required later, enforce authentication/authorization through the corporate Ingress/API Gateway; do not expose the monitor directly to the Internet. + +--- + +## 20. How to release a new version + +Recommended flow: + +### 1. Update code/configuration + +Never commit real credentials. + +### 2. Run tests + +```bash +python -m pytest -v +python -m compileall evaluator +``` + +### 3. Test locally with a small limit + +```bash +python -m evaluator.cli run-agents \ + --agent-id \ + --limit 5 +``` + +### 4. Validate the run + +```bash +python -m evaluator.cli runs --limit 10 +python -m evaluator.cli progress --events 100 +``` + +### 5. Confirm GCS + +The event stream must include: + +```text +GCS_UPLOADED +``` + +with a URI such as: + +```text +gs:////AGENTE__LLM_JUDGE_.TXT.GZ +``` + +### 6. Update environment secrets/configuration + +```bash +./deploy/scripts/create-secrets.sh +``` + +### 7. Build/push a new tag + +```bash +export REGISTRY= +export IMAGE_TAG= +./deploy/scripts/build-push.sh +``` + +### 8. Deploy + +```bash +./deploy/scripts/deploy.sh +``` + +### 9. Trigger a manual certification run + +```bash +./deploy/scripts/run-now.sh +``` + +### 10. Validate logs and artifacts + +```bash +kubectl -n agent-evaluator- logs -f job/ +``` + +Also confirm the GCS object and the `judge-job-{agentId}` audit in Langfuse. + +--- + +## 21. Troubleshooting + +### `Collected 0 traces` + +Check, in this order: + +1. processed date (`daysBack` uses BRT); +2. Langfuse credentials for the correct `agentId`; +3. Langfuse project `host`; +4. `langfuse_agent_aliases`; +5. whether sessions/traces exist in the period; +6. whether the logical `agentId` differs from `metadata.agent_id` in traces. + +Example: + +```yaml +- agentId: billing + langfuse_agent_aliases: + - telecom_contas + - billing_agent +``` + +### `No Langfuse credentials configured for agent ...` + +The agent exists in `agents.yaml` but not in the file referenced by: + +```env +LANGFUSE_CREDENTIALS_CONFIG_PATH +``` + +Add an entry with the same `agentId`. + +### `Langfuse credentials definition file not found` + +Create/mount: + +```text +configs/langfuse/langfuse_credentials.yaml +``` + +or configure another path. + +### `TIM specification requires GCS upload...` + +`run-agents` was executed with: + +```env +ENABLE_GCS_UPLOAD=false +``` + +Set it to `true` and configure GCP credentials. + +### GCS upload failure + +Check: + +```bash +echo "$GOOGLE_APPLICATION_CREDENTIALS" +ls -l "$GOOGLE_APPLICATION_CREDENTIALS" +``` + +Also validate the bucket resolved through `APP_ENV`/`storage.yaml`. + +### OCI SDK dedicated endpoint failure + +Check: + +```env +LLM_PROVIDER=oci_sdk +OCI_GENAI_BASE_URL=https:// +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... +OCI_COMPARTMENT_ID=... +``` + +Do not use `/openai/v1` in the native `oci_sdk` endpoint. + +### Database schema missing + +```bash +python -m evaluator.cli init-db +``` + +### View progress without a UI + +The service exposes no HTTP endpoint. Use: + +```bash +python -m evaluator.cli runs +python -m evaluator.cli progress --events 100 +``` + +--- + +## 22. Main environment variables + +| Variable | Purpose | +|---|---| +| `APP_ENV` | environment used to resolve storage | +| `ADB_*` | Oracle connection and wallet | +| `ENABLE_LANGFUSE` | enables Langfuse collection | +| `LANGFUSE_CREDENTIALS_CONFIG_PATH` | declarative per-agent YAML | +| `LANGFUSE_*` | global fallback only for manual `run` | +| `PUBLISH_LANGFUSE_SCORES` | publishes trace scores back to Langfuse | +| `LLM_PROVIDER` | `mock`, `oci_openai`, `oci_sdk`, `openai_compatible` | +| `LLM_PROFILE` | judge profile | +| `OCI_AUTH_MODE` | OCI SDK authentication mode | +| `OCI_GENAI_BASE_URL` | OCI GenAI endpoint | +| `OCI_GENAI_MODEL` | model ID/name or endpoint-OCID compatibility | +| `OCI_GENAI_ENDPOINT_ID` | dedicated endpoint OCID | +| `OCI_COMPARTMENT_ID` | OCI compartment | +| `AGENTS_CONFIG_PATH` | agents file | +| `TRACE_PROMPT_PATH` | trace-level prompt | +| `SESSION_PROMPT_PATH` | session-level prompt | +| `BATCH_SIZE` | internal processing batch size | +| `MAX_ATTEMPTS` | configurable attempt limit | +| `CONTEXT_TURN_LIMIT` | last N turns passed to trace judge | +| `ENABLE_GCS_UPLOAD` | requires upload in production flow | +| `STORAGE_CONFIG_PATH` | bucket mapping by environment | +| `JUDGE_GCS_BUCKET` | bucket override | +| `GOOGLE_APPLICATION_CREDENTIALS` | GCP credentials | +| `OUTPUT_DIR` | temporary export directory | + +--- + +## 23. Security and operations + +- do not commit the real `.env`; +- do not commit the real `langfuse_credentials.yaml`; +- do not commit GCP service account JSON; +- do not commit Oracle Wallet files; +- in OKE, prefer `oke_workload_identity` where applicable; +- mount credentials as Kubernetes Secrets; +- keep `readOnlyRootFilesystem` enabled in the CronJob; +- keep `concurrencyPolicy: Forbid` to prevent two D+1 jobs from the same deployment running concurrently; +- never run `reset-db` in production without an explicit recovery plan. + +--- + +## 24. Certification checklist + +Before promoting a version: + +- [ ] `pip check` passes; +- [ ] `python -m pytest -v` passes; +- [ ] `python -m compileall evaluator` passes; +- [ ] `show-config` points to the expected ADB/OCI/config files; +- [ ] `agents.yaml` contains only the expected agents for that deployment; +- [ ] Langfuse credentials exist for every enabled `agentId`; +- [ ] `run-agents --limit 5` finds traces; +- [ ] sampling operates on sessions, not individual traces; +- [ ] trace/session judges complete; +- [ ] output contains the 12-column header; +- [ ] filename matches `AGENTE_{agentId}_LLM_JUDGE_{YYYYMMDD}_{HHMMSS}.TXT.GZ`; +- [ ] `GCS_UPLOADED` event is confirmed; +- [ ] local file is deleted after upload; +- [ ] Langfuse `created/completed` audit is confirmed; +- [ ] CronJob has `suspend: false`; +- [ ] CronJob timezone is `America/Sao_Paulo`; +- [ ] no real credentials are present in the repository. + +--- + +## 25. Quick mental model for new contributors + +If you need to understand the project quickly, think of it as: + +```text +Langfuse stores conversations + ↓ +Evaluator selects D+1 sessions + ↓ +OCI GenAI judges traces and sessions + ↓ +ADB stores execution state and results + ↓ +GCS receives the official artifact + ↓ +Langfuse stores the batch audit trace +``` + +Extension points are configuration-driven: + +```text +Which agent? -> configs/judge/agents.yaml +Which Langfuse project?-> langfuse_credentials.yaml +Which bucket? -> configs/judge/storage.yaml +Which prompt? -> trace_metrics.yaml / session_metrics.yaml +Which model? -> .env + llm_profiles.yaml +Which identity rules? -> configs/identity.yaml +``` + +The most important architectural rule is: **the evaluator code must remain generic; agent, environment, credential, and storage details belong in configuration.** diff --git a/README.md b/README.md new file mode 100644 index 0000000..3c798cc --- /dev/null +++ b/README.md @@ -0,0 +1,1551 @@ +# Agent Framework OCI Evaluator — LLM-as-a-Judge + +**Idioma:** Português | [English](README.en-US.md) + +## 1. Visão geral + +O **Agent Framework OCI Evaluator** é o componente batch responsável por avaliar, de forma assíncrona, a qualidade das conversas produzidas pelos agentes construídos sobre o Agent Framework OCI. + +O serviço lê sessões e traces do **Langfuse v3**, seleciona uma amostra determinística de sessões, calcula métricas de qualidade com um **LLM-as-a-Judge**, persiste resultados em **Oracle Autonomous Database**, gera o arquivo oficial `.TXT.GZ`, publica esse arquivo em **Google Cloud Storage (GCS)** e registra a execução do próprio job no Langfuse. + +Ele foi desenhado para execução **D+1**, como um **Kubernetes CronJob**, sem endpoint HTTP/gRPC de runtime. + +> **Modelo de implantação recomendado:** uma instância do evaluator por agente/squad. O código é genérico e suporta múltiplos agentes na configuração, mas em produção a recomendação é cada deployment carregar apenas o agente de sua responsabilidade. Isso mantém credenciais, ownership, scheduling e troubleshooting isolados. + +--- + +## 2. Por que este componente existe + +Avaliar respostas generativas em tempo real dentro do fluxo síncrono do agente aumenta latência, custo e acoplamento. O evaluator separa essa responsabilidade do atendimento online. + +O desenho é: + +```text +Atendimento online + | + v +Agent Framework + | + +--------------------> Langfuse v3 + | + | traces / sessions + v + Evaluator D+1 (CronJob) + | + +------------------+------------------+ + | | | + v v v + OCI GenAI ADB GCS + LLM-as-a-Judge estado/resultados .TXT.GZ oficial + | + +-----------------------> Langfuse + auditoria / scores opcionais +``` + +Essa separação traz alguns benefícios: + +- não adiciona latência ao atendimento; +- permite reprocessamento e auditoria; +- permite amostragem configurável por agente; +- isola falhas de avaliação do agente online; +- permite utilizar modelos dedicados do OCI GenAI; +- mantém um artefato oficial para a camada analítica da TIM; +- evita acoplamento do código a nomes específicos de agentes, buckets ou credenciais. + +--- + +## 3. Conceitos principais + +### 3.1. Trace + +Um **trace** representa uma interação avaliada individualmente. Para cada trace são calculadas métricas trace-level: + +- `judgeScore` — qualidade global da resposta; +- `accuracyScore` — aderência/factualidade em relação ao contexto; +- `alucinationScore` — grau de afirmações não suportadas pelo contexto; +- `promptLength` — tamanho bruto, em caracteres, do `trace.input`. + +### 3.2. Session + +Uma **session** representa a conversa completa. Uma sessão normalmente possui vários traces. + +As métricas session-level são calculadas uma vez e replicadas nas linhas dos traces daquela sessão: + +- `inferredCsiScore`; +- `resolution`; +- `conversationPrecision`; +- `loop`. + +### 3.3. Amostragem por sessão + +A amostragem é feita sobre **sessions**, nunca sobre traces isolados. + +A seed é determinística: + +```text +agentId:reference_date +``` + +Exemplo: + +```text +billing:2026-08-13 +``` + +Com isso, duas execuções para o mesmo agente, data e percentual selecionam as mesmas sessões. + +Depois que uma sessão é selecionada, **todos os traces daquela sessão** são mantidos. + +### 3.4. Janela BRT + +`run-agents` calcula a data de referência com base em: + +```text +D - daysBack +``` + +usando explicitamente: + +```text +America/Sao_Paulo +``` + +Com `daysBack: 1`, uma execução em 14/08 avalia o dia 13/08 completo, de 00:00:00 até 23:59:59.999999 BRT. + +### 3.5. CONTEXT_TURN_LIMIT + +`CONTEXT_TURN_LIMIT` controla quantos turnos anteriores são enviados ao judge trace-level. + +Default: + +```env +CONTEXT_TURN_LIMIT=6 +``` + +Ele existe porque uma resposta não deve ser julgada fora do contexto da conversa. Aumentar o valor fornece mais contexto, mas também aumenta tokens, custo e latência do judge. + +### 3.6. Loop + +A métrica `loop` não mede repetição textual simples. Ela representa o maior **streak consecutivo de `metadata.intent`** dentro da sessão. + +Exemplo: + +```text +billing +billing +billing +payment +payment +``` + +Resultado: + +```text +loop = 3 +``` + +--- + +## 4. Fluxo completo de execução + +O fluxo produtivo é executado pelo comando: + +```bash +python -m evaluator.cli run-agents +``` + +ou pelo entrypoint instalado: + +```bash +af-evaluator run-agents +``` + +Etapas: + +```text +1. Carrega agents.yaml +2. Calcula D-daysBack em BRT +3. Resolve credenciais Langfuse do agente +4. Busca TODAS as sessions nativas do período, com paginação +5. Busca TODOS os traces do período, com paginação +6. Busca observations em bulk/paginado +7. Recupera session_id real via metadata quando o trace legado não possui sessionId nativo +8. Filtra traces do agente/aliases +9. Faz sampling determinístico das sessions lógicas +10. Mantém todos os traces das sessions selecionadas +11. Persiste itens no ADB +12. Executa judge trace-level +13. Executa judge session-level +14. Calcula loop local +15. Gera .TXT.GZ +16. Faz upload para GCS +17. Remove o arquivo local após sucesso +18. Atualiza auditoria judge-job-{agentId} no Langfuse +``` + +### 4.1. Por que a coleta não usa N+1 + +A implementação evita o padrão antigo: + +```text +1 trace + -> GET trace detail + -> GET observations +``` + +repetido para cada trace. + +Agora a coleta é paginada em lote: + +```text +/api/public/sessions page 1..N +/api/public/traces page 1..N +/api/public/observations page 1..N +``` + +Assim, 10.000 traces não produzem ~20.000 chamadas adicionais. O número de requests cresce com o número de páginas da API. + +O `PAGE_SIZE` atual é 100. + +### 4.2. Compatibilidade com traces sem `sessionId` nativo + +Algumas instrumentações históricas registram a sessão no metadata do trace/observation, mas não promovem o valor para o campo nativo `trace.sessionId` do Langfuse. O evaluator recupera a sessão real com a seguinte precedência: + +```text +trace.sessionId / trace.session_id + -> trace.metadata.session_id / sessionId / conversation_key + -> observation.metadata.session_id / sessionId / conversation_key + -> business_context.session_key (último fallback de metadata) +``` + +O `trace.id` **nunca** é usado como `session_id`. Assim, o evaluator preserva sampling e métricas de session mesmo para traces históricos, sem transformar cada trace em uma sessão artificial. As estatísticas de coleta expõem `native_sessions_read`, `metadata_sessions_recovered` e `traces_without_session`. + +--- + +## 5. Arquitetura de componentes + +```text +agent_framework_oci_evaluator/ +├── evaluator/ +│ ├── analytics/ +│ │ └── vloop.py +│ ├── collectors/ +│ │ ├── langfuse.py +│ │ ├── langfuse_credentials.py +│ │ ├── agent_framework.py +│ │ └── mock.py +│ ├── config/ +│ │ ├── agents.py +│ │ └── settings.py +│ ├── core/ +│ │ └── models.py +│ ├── identity/ +│ │ └── resolver.py +│ ├── judges/ +│ │ └── llm_judge.py +│ ├── llm/ +│ │ ├── client.py +│ │ └── profile_resolver.py +│ ├── output/ +│ │ └── legacy_exporter.py +│ ├── persistence/ +│ │ ├── oracle_store.py +│ │ └── repository.py +│ ├── prompts/ +│ │ └── loader.py +│ ├── publishers/ +│ │ ├── gcs_export.py +│ │ ├── langfuse_job_audit.py +│ │ └── langfuse_scores.py +│ ├── cli.py +│ ├── engine.py +│ └── time_utils.py +├── configs/ +│ ├── identity.yaml +│ ├── judge/ +│ │ ├── agents.yaml +│ │ ├── storage.yaml +│ │ ├── trace_metrics.yaml +│ │ └── session_metrics.yaml +│ ├── langfuse/ +│ │ └── langfuse_credentials.example.yaml +│ └── llm_profiles/ +│ └── llm_profiles.yaml +├── deploy/ +├── tests/ +├── .env.example +├── Dockerfile +└── pyproject.toml +``` + +### 5.1. Collectors + +Transformam fontes externas em `ConversationRecord`. + +Fontes disponíveis: + +```text +langfuse +agent_framework +mock +``` + +`langfuse` é a fonte produtiva padrão. + +### 5.2. EvaluationEngine + +`evaluator/engine.py` coordena o pipeline: + +- criação do run; +- coleta; +- amostragem; +- persistência; +- judge; +- export; +- upload; +- auditoria. + +### 5.3. Persistence + +Os estados e resultados são persistidos em Oracle Database. O schema é criado automaticamente pelo comando `init-db`. + +As principais tabelas são: + +```text +EVALUATION_RUN +EVALUATION_PROGRESS_EVENT +EVALUATION_ITEM +EVALUATION_RESULT +EVALUATION_METRIC +EVALUATION_FINDING +``` + +O prefixo é configurável por: + +```env +ADB_TABLE_PREFIX=AGENTFW +``` + +### 5.4. Publishers + +- `gcs_export.py` — upload do artefato oficial; +- `langfuse_job_audit.py` — auditoria `judge-job-{agentId}`; +- `langfuse_scores.py` — publicação opcional de scores de trace. + +> **Nota:** a auditoria produtiva do job usa as credenciais dinâmicas do agente. A publicação opcional de scores (`PUBLISH_LANGFUSE_SCORES=true`) ainda utiliza as credenciais globais `LANGFUSE_HOST/PUBLIC_KEY/SECRET_KEY`; deixe-a desabilitada quando não houver um projeto global apropriado. + +--- + +## 6. Configuração de agentes + +Arquivo padrão: + +```text +configs/judge/agents.yaml +``` + +Exemplo: + +```yaml +agents: + - agentId: billing + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: + - telecom_contas + gcs_prefix: agnt_ai_contas +``` + +Campos: + +| Campo | Descrição | +|---|---| +| `agentId` | identidade lógica oficial do agente | +| `enabled` | habilita/desabilita o agente naquele deployment | +| `daysBack` | quantos dias voltar a partir do dia atual BRT | +| `percentage` | percentual de sessions avaliadas, de 0.0 a 1.0 | +| `langfuse_agent_aliases` | nomes técnicos aceitos para o agente nos traces; quando preenchido, funciona como filtro explícito independentemente da origem das credenciais Langfuse | +| `gcs_prefix` | prefixo do agente dentro do bucket | + +Também são aceitos `agent_id` e `days_back` por compatibilidade, mas o formato recomendado é `agentId` / `daysBack`. + +**Semântica de `langfuse_agent_aliases`:** quando a lista possui valores, somente traces cujo identificador de agente corresponda a um dos aliases são elegíveis para sampling e avaliação. Quando a lista está vazia, o evaluator não aplica filtro por identidade de agente e considera o projeto/endpoint Langfuse configurado exclusivo para aquela entrada. + +### 6.1. Um evaluator por agente + +Para o modelo recomendado de uma instância por agente, mantenha apenas uma entrada habilitada naquele deployment: + +```yaml +agents: + - agentId: billing + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: + - telecom_contas + - billing_agent + gcs_prefix: agnt_ai_contas +``` + +O mesmo código pode ser usado em outro repositório/deployment apenas trocando os arquivos de configuração. + +--- + +## 7. Credenciais Langfuse dinâmicas + +Não existem variáveis hardcoded do tipo: + +```text +LANGFUSE_BILLING_* +LANGFUSE_SALES_* +``` + +O evaluator lê uma definição declarativa: + +```env +LANGFUSE_CREDENTIALS_CONFIG_PATH=configs/langfuse/langfuse_credentials.yaml +``` + +Crie o arquivo a partir do exemplo: + +```bash +cp configs/langfuse/langfuse_credentials.example.yaml \ + configs/langfuse/langfuse_credentials.yaml +``` + +Exemplo: + +```yaml +langfuse: + agents: + - agentId: billing + host: https://langfuse.example.internal + publicKey: pk-... + secretKey: sk-... + + - agentId: outro-agente + host: https://outro-langfuse.internal + publicKey: pk-... + secretKey: sk-... +``` + +O resolver percorre todas as entradas. Adicionar um agente não exige alteração Python. + +### 7.1. Referências a env ou arquivos + +Para não manter secrets no Git: + +```yaml +langfuse: + agents: + - agentId: billing + hostEnv: BILLING_LF_HOST + publicKeyEnv: BILLING_LF_PUBLIC + secretKeyFile: /var/secrets/langfuse/billing-secret-key +``` + +Campos suportados: + +```text +host / hostEnv / hostFile +publicKey / publicKeyEnv / publicKeyFile +secretKey / secretKeyEnv / secretKeyFile +``` + +Ordem de resolução de cada campo: + +```text +valor direto -> variável de ambiente declarada -> arquivo declarado +``` + +### 7.2. Credenciais globais + +Estas variáveis continuam disponíveis apenas para o comando manual genérico `run`: + +```env +LANGFUSE_HOST=http://localhost:3005 +LANGFUSE_PUBLIC_KEY=... +LANGFUSE_SECRET_KEY=... +``` + +O fluxo produtivo `run-agents` resolve credenciais pelo arquivo declarativo por agente. + +--- + +## 8. Configuração do LLM + +Providers aceitos: + +```text +mock +oci_openai +oci_sdk +openai_compatible +``` + +A recomendação para os modelos OCI dedicados é: + +```env +LLM_PROVIDER=oci_sdk +``` + +### 8.1. OCI SDK + +Exemplo local com `~/.oci/config`: + +```env +LLM_PROVIDER=oci_sdk +OCI_AUTH_MODE=config_file +OCI_CONFIG_FILE=~/.oci/config +OCI_PROFILE=DEFAULT +OCI_REGION=sa-saopaulo-1 +OCI_COMPARTMENT_ID=ocid1.compartment... +OCI_GENAI_BASE_URL=https://inference.generativeai.sa-saopaulo-1.oci.oraclecloud.com +OCI_GENAI_MODEL=meta.llama-3.3-70b-instruct +``` + +Modos de autenticação suportados pelo código: + +```text +config_file +instance_principal +resource_principal +oke_workload_identity +workload_identity +``` + +### 8.2. Dedicated endpoint + +Para endpoint dedicado: + +```env +OCI_GENAI_BASE_URL=https://.inference.generativeai..oci.oraclecloud.com +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... +``` + +Também é aceito o OCID do dedicated endpoint em `OCI_GENAI_MODEL`. + +O provider seleciona `DedicatedServingMode` quando há endpoint OCID; caso contrário usa `OnDemandServingMode`. + +### 8.3. Perfis do judge + +Arquivo: + +```text +configs/llm_profiles/llm_profiles.yaml +``` + +Exemplo: + +```yaml +profiles: + judge: + temperature: 0 + max_tokens: 900 +``` + +Selecionado por: + +```env +LLM_PROFILE=judge +``` + +--- + +## 9. Prompts e métricas + +### 9.1. Trace metrics + +Arquivo: + +```text +configs/judge/trace_metrics.yaml +``` + +O judge deve retornar JSON com: + +```json +{ + "judgeScore": 0.0, + "accuracyScore": 0.0, + "alucinationScore": 0.0, + "rationale": "..." +} +``` + +Se o parse falhar: + +```text +1ª tentativa + -> parse falha +2ª tentativa com retry-hint + -> parse falha novamente +métricas trace-level = None + -> exportadas como "0.00" +``` + +### 9.2. Session metrics + +Arquivo: + +```text +configs/judge/session_metrics.yaml +``` + +Resultado esperado: + +```json +{ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "..." +} +``` + +Os valores session-level são replicados no arquivo para todos os traces da mesma session. + +--- + +## 10. Identity normalization + +Arquivo: + +```text +configs/identity.yaml +``` + +A camada de identidade converte identificadores específicos de canais/domínios para chaves canônicas como: + +```text +customer_key +contract_key +interaction_key +account_key +resource_key +session_key +``` + +Isso evita espalhar lógica como `cpf`, `msisdn`, `invoice_id`, `order_id`, `ura_call_id` etc. pelo código do evaluator. + +A ordem das fontes funciona como fallback. + +Exemplo: + +```yaml +session_key: + sources: + - business_context.session_key + - session_key + - conversation_key + - session_id +``` + +--- + +## 11. Configuração de GCS + +Buckets ficam fora do código em: + +```text +configs/judge/storage.yaml +``` + +Exemplo: + +```yaml +storage: + buckets: + dev: tim-bigdata-dev-ca1f-raw-standard-agnt-ai-atendimento + fqa: tim-bigdata-fqa-60ce-raw-standard-agnt-ai-atendimento + prd: tim-bigdata-prod-e305-raw-standard-agnt-ai-atendimento +``` + +Seleção: + +```env +APP_ENV=dev +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +``` + +Override de emergência/operação: + +```env +JUDGE_GCS_BUCKET=meu-bucket +``` + +Para habilitar o fluxo produtivo: + +```env +ENABLE_GCS_UPLOAD=true +GOOGLE_APPLICATION_CREDENTIALS=/path/GCP_ACCESS_KEY.json +``` + +`run-agents` considera GCS obrigatório. Se `ENABLE_GCS_UPLOAD=false` ou o upload falhar, o run do agente termina como `FAILED`. + +--- + +## 12. Arquivo de saída + +Padrão: + +```text +AGENTE_{agentId}_LLM_JUDGE_{YYYYMMDD}_{HHMMSS}.TXT.GZ +``` + +Exemplo: + +```text +AGENTE_billing_LLM_JUDGE_20260814_020015.TXT.GZ +``` + +Cabeçalho: + +```text +"judgeScore"|;"accuracyScore"|;"alucinationScore"|;"promptLength"|;"loop"|;"inferredCsiScore"|;"resolution"|;"conversationPrecision"|;"uraCallId"|;"channelId"|;"sessionId"|;"messageId" +``` + +Linha final: + +```text +"TOTAL"|;"" +``` + +Após upload bem-sucedido para GCS, o arquivo local é apagado. + +--- + +## 13. Instalação local + +### 13.1. Pré-requisitos + +- Python 3.11+; +- acesso ao Oracle Database/Autonomous Database; +- Oracle Wallet quando aplicável; +- acesso ao Langfuse v3; +- acesso ao OCI Generative AI para execução real do judge; +- credencial GCP para upload produtivo. + +### 13.2. Criar venv + +Linux/WSL/macOS: + +```bash +python -m venv .venv +source .venv/bin/activate +``` + +Windows PowerShell: + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +``` + +### 13.3. Instalar + +Modo recomendado durante desenvolvimento: + +```bash +python -m pip install --upgrade pip setuptools wheel +pip install -e . +``` + +Validar: + +```bash +pip check +python -c "import oci, httpx, oracledb; import google.cloud.storage; print('Dependencias OK')" +``` + +--- + +## 14. Configuração local passo a passo + +### Passo 1 — `.env` + +```bash +cp .env.example .env +``` + +Edite os valores reais. + +Configuração mínima típica: + +```env +APP_ENV=dev + +ADB_USER=admin +ADB_PASSWORD=... +ADB_DSN=... +ADB_WALLET_LOCATION=/path/to/wallet +ADB_WALLET_PASSWORD=... +ADB_TABLE_PREFIX=AGENTFW + +ENABLE_LANGFUSE=true +LANGFUSE_CREDENTIALS_CONFIG_PATH=configs/langfuse/langfuse_credentials.yaml + +LLM_PROVIDER=oci_sdk +LLM_PROFILE=judge +OCI_AUTH_MODE=config_file +OCI_CONFIG_FILE=~/.oci/config +OCI_PROFILE=DEFAULT +OCI_REGION=sa-saopaulo-1 +OCI_COMPARTMENT_ID=... +OCI_GENAI_BASE_URL=https://... +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... + +CONTEXT_TURN_LIMIT=6 +BATCH_SIZE=50 +MAX_ATTEMPTS=3 + +ENABLE_GCS_UPLOAD=true +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +GOOGLE_APPLICATION_CREDENTIALS=/path/GCP_ACCESS_KEY.json +``` + +### Passo 2 — Langfuse + +```bash +cp configs/langfuse/langfuse_credentials.example.yaml \ + configs/langfuse/langfuse_credentials.yaml +``` + +Declare o agente que será executado. + +### Passo 3 — agents.yaml + +Ajuste: + +```text +configs/judge/agents.yaml +``` + +Para uma instância dedicada, deixe somente o agente daquela squad habilitado. + +### Passo 4 — validar configuração + +```bash +python -m evaluator.cli show-config +``` + +### Passo 5 — inicializar banco + +```bash +python -m evaluator.cli init-db +``` + +> `reset-db` apaga e recria o schema do evaluator. Use apenas em ambiente controlado. + +--- + +## 15. CLI + +A ajuda real da versão atual é: + +```bash +python -m evaluator.cli --help +``` + +Comandos disponíveis: + +```text +init-db +reset-db +show-config +run +run-agents +progress +runs +``` + +### 15.1. `run` — execução manual/diagnóstica + +Sintaxe: + +```bash +python -m evaluator.cli run PERIOD_START PERIOD_END [OPTIONS] +``` + +Exemplo: + +```bash +python -m evaluator.cli run \ + 2026-08-12T00:00:00 \ + 2026-08-12T23:59:59 \ + --source langfuse \ + --limit 10 +``` + +Opções: + +```text +--source TEXT default: langfuse +--limit INTEGER +--show-progress / --no-show-progress +``` + +**Importante:** `run` é um caminho manual genérico. Ele coleta, avalia e persiste resultados, mas não executa o contrato produtivo completo por agente (sampling configurado, export GCS e auditoria de job). Para homologação/produção use `run-agents`. + +### 15.2. `run-agents` — fluxo produtivo + +```bash +python -m evaluator.cli run-agents +``` + +Um agente: + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --source langfuse +``` + +Teste limitado: + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --limit 10 +``` + +Opções: + +```text +--source TEXT default: langfuse +--agent-id TEXT +--limit INTEGER +``` + +`run-agents` usa `daysBack`, `percentage`, credenciais e `gcs_prefix` do agente. + +### 15.3. Listar runs + +```bash +python -m evaluator.cli runs +``` + +ou: + +```bash +python -m evaluator.cli runs --limit 50 +``` + +### 15.4. Progresso de um run + +```bash +python -m evaluator.cli progress +``` + +Com mais eventos: + +```bash +python -m evaluator.cli progress --events 100 +``` + +Eventos típicos do fluxo produtivo: + +```text +LANGFUSE_JOB_AUDIT +RUN_CREATED +COLLECTING +COLLECTED +SAMPLED +ITEMS_INSERTED +BATCH_STARTED +ITEM_COMPLETED +SESSION_JUDGE_COMPLETED +COMPLETED +EXPORTED +GCS_UPLOADED +LOCAL_FILE_DELETED +LANGFUSE_JOB_AUDIT +``` + +--- + +## 16. Testes + +Instale `pytest` caso necessário: + +```bash +pip install pytest +``` + +### 16.1. Todos os testes + +```bash +python -m pytest -v +``` + +### 16.2. Conformidade TIM + +```bash +python -m pytest tests/test_tim_spec.py -v +``` + +### 16.3. Conformidade ampliada + +```bash +python -m pytest tests/test_full_tim_compliance.py -v +``` + +### 16.4. Regressão de hardcodes + +```bash +python -m pytest tests/test_no_product_hardcodes.py -v +``` + +### 16.5. Suite recomendada antes de um release + +```bash +python -m pytest \ + tests/test_tim_spec.py \ + tests/test_full_tim_compliance.py \ + tests/test_no_product_hardcodes.py \ + -v +``` + +Validar sintaxe/imports: + +```bash +python -m compileall evaluator +``` + +--- + +## 17. Teste funcional local recomendado + +### 17.1. Smoke test com mock + +Configure: + +```env +LLM_PROVIDER=mock +``` + +E rode um período pequeno com source mock quando desejado: + +```bash +python -m evaluator.cli run \ + 2026-08-12T00:00:00 \ + 2026-08-12T23:59:59 \ + --source mock +``` + +### 17.2. Teste Langfuse sem volume grande + +```bash +python -m evaluator.cli run-agents \ + --agent-id billing \ + --limit 5 +``` + +Depois: + +```bash +python -m evaluator.cli runs --limit 5 +python -m evaluator.cli progress --events 100 +``` + +Confirme: + +- `COLLECTED` > 0; +- `SAMPLED` mostra sessions selecionadas; +- itens concluem sem `ITEM_FAILED`; +- `EXPORTED` aparece; +- `GCS_UPLOADED` contém URI `gs://...`; +- `LOCAL_FILE_DELETED` aparece; +- auditoria `completed` foi confirmada. + +--- + +## 18. Docker + +Build local: + +```bash +docker build -t agent-framework-evaluator:local . +``` + +O runtime produtivo é batch. O container não precisa expor porta HTTP. + +Para Kubernetes, o `CronJob` chama: + +```text +python -m evaluator.cli run-agents --source langfuse +``` + +--- + +## 19. Kubernetes / OKE + +O **evaluator propriamente dito continua CronJob-only**, conforme a especificação: ele não abre porta HTTP/gRPC e o processamento continua sendo executado exclusivamente pelo `evaluator-batch`. + +A interface web foi reintroduzida como um **componente separado e somente leitura**, `evaluator-monitor`. Ela não executa avaliação, não altera runs e não muda o contrato batch do evaluator. O monitor apenas consulta as tabelas de execução/progresso no Autonomous Database. + +```text +Evaluator batch (evaluator-batch CronJob) + │ + ├── cria/atualiza EVALUATION_RUN + ├── grava EVALUATION_PROGRESS_EVENT + └── grava EVALUATION_RESULT + │ + ▼ + Autonomous Database + ▲ + │ somente leitura + Evaluator Monitor (Deployment HTTP) + │ + └── /ui + APIs de acompanhamento +``` + +Recursos Kustomize: + +```text +Namespace +ServiceAccount +ConfigMap +CronJob evaluator-batch +Deployment evaluator-monitor +Service evaluator-monitor (ClusterIP) +``` + +Não existe Ingress por default. Assim, a interface não é exposta para fora do cluster automaticamente. + +CronJob atual: + +```yaml +schedule: "0 2 * * *" +timeZone: "America/Sao_Paulo" +suspend: false +concurrencyPolicy: Forbid +``` + +### 19.1. Preparar arquivos de deploy + +```bash +cp deploy/env/evaluator.env.example deploy/env/evaluator.env +cp deploy/langfuse/langfuse_credentials.example.yaml \ + deploy/langfuse/langfuse_credentials.yaml +``` + +Coloque o wallet em: + +```text +deploy/wallet/ +``` + +Credencial GCP: + +```text +deploy/gcp/GCP_ACCESS_KEY.json +``` + +### 19.2. Criar secrets + +DEV: + +```bash +./deploy/scripts/create-secrets.sh dev +``` + +HML: + +```bash +./deploy/scripts/create-secrets.sh hml +``` + +PROD: + +```bash +./deploy/scripts/create-secrets.sh prod +``` + +São criados/atualizados: + +```text +evaluator-secrets +evaluator-wallet +evaluator-langfuse +evaluator-gcp (quando o arquivo GCP existir) +``` + +### 19.3. Build e push + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG= +./deploy/scripts/build-push.sh prod +``` + +### 19.4. Deploy + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG= +./deploy/scripts/deploy.sh prod +``` + +Por default o script também executa o job de inicialização do banco. + +Para não executar: + +```bash +export RUN_INIT_DB=false +``` + +### 19.5. Status + +```bash +./deploy/scripts/status.sh prod +``` + +ou diretamente: + +```bash +kubectl -n agent-evaluator-prod get cronjob,jobs,pods +``` + +### 19.6. Execução imediata fora do horário + +```bash +./deploy/scripts/run-now.sh prod +``` + +O script cria um Job baseado no CronJob. + +Acompanhar: + +```bash +kubectl -n agent-evaluator-prod get jobs,pods +kubectl -n agent-evaluator-prod logs -f job/ +``` + + +### 19.7. Interface web de acompanhamento + +A UI é servida pelo componente independente `evaluator-monitor`: + +```text +GET /ui +GET /health +GET /api/runs +GET /api/runs/{run_id} +GET /api/runs/{run_id}/results +``` + +Para acessar localmente um monitor implantado no Kubernetes sem criar Ingress: + +```bash +kubectl -n agent-evaluator-prod port-forward svc/evaluator-monitor 8080:8080 +``` + +Depois abra: + +```text +http://localhost:8080/ui +``` + +A tela mostra runs recentes, `RUN_ID`, agente, status, progresso, eventos e resultados. O browser faz auto-refresh a cada 5 segundos. + +Para executar o monitor localmente: + +```bash +python -m uvicorn evaluator_monitor.app:app --host 0.0.0.0 --port 8080 +``` + +O monitor utiliza as mesmas configurações `ADB_*`/wallet do evaluator. Ele é **read-only** do ponto de vista funcional: não há endpoint para iniciar, cancelar, apagar ou alterar runs. + +> Recomendação de segurança: mantenha o Service como `ClusterIP`. Se futuramente houver necessidade de exposição externa, coloque autenticação/autorização no Ingress/API Gateway corporativo; não exponha o monitor diretamente à Internet. + +--- + +## 20. Como subir uma nova versão + +Fluxo recomendado: + +### 1. Atualizar código/configuração + +Não coloque credenciais reais no Git. + +### 2. Rodar testes + +```bash +python -m pytest -v +python -m compileall evaluator +``` + +### 3. Testar localmente com limite + +```bash +python -m evaluator.cli run-agents \ + --agent-id \ + --limit 5 +``` + +### 4. Validar o run + +```bash +python -m evaluator.cli runs --limit 10 +python -m evaluator.cli progress --events 100 +``` + +### 5. Confirmar GCS + +O evento deve conter: + +```text +GCS_UPLOADED +``` + +com uma URI como: + +```text +gs:////AGENTE__LLM_JUDGE_.TXT.GZ +``` + +### 6. Atualizar secrets/configuração do ambiente + +```bash +./deploy/scripts/create-secrets.sh +``` + +### 7. Build/push com uma tag nova + +```bash +export REGISTRY= +export IMAGE_TAG= +./deploy/scripts/build-push.sh +``` + +### 8. Deploy + +```bash +./deploy/scripts/deploy.sh +``` + +### 9. Rodar manualmente para homologação + +```bash +./deploy/scripts/run-now.sh +``` + +### 10. Verificar logs e artefatos + +```bash +kubectl -n agent-evaluator- logs -f job/ +``` + +Confirme também o objeto no bucket GCS e a auditoria `judge-job-{agentId}` no Langfuse. + +--- + +## 21. Troubleshooting + +### `Collected 0 traces` + +Verifique nesta ordem: + +1. data processada (`daysBack` usa BRT); +2. credencial Langfuse do `agentId` correto; +3. `host` do projeto Langfuse; +4. `langfuse_agent_aliases`; +5. existência de sessions/traces naquele período; +6. se o `agentId` lógico é diferente do `metadata.agent_id` dos traces. + +Exemplo: + +```yaml +- agentId: billing + langfuse_agent_aliases: + - telecom_contas + - billing_agent +``` + +### `No Langfuse credentials configured for agent ...` + +O agente existe em `agents.yaml`, mas não no arquivo apontado por: + +```env +LANGFUSE_CREDENTIALS_CONFIG_PATH +``` + +Adicione uma entrada com o mesmo `agentId`. + +### `Langfuse credentials definition file not found` + +Crie/mapeie o arquivo: + +```text +configs/langfuse/langfuse_credentials.yaml +``` + +ou configure outro path. + +### `TIM specification requires GCS upload...` + +`run-agents` foi executado com: + +```env +ENABLE_GCS_UPLOAD=false +``` + +Altere para `true` e configure a credencial GCP. + +### GCS falha + +Verifique: + +```bash +echo "$GOOGLE_APPLICATION_CREDENTIALS" +ls -l "$GOOGLE_APPLICATION_CREDENTIALS" +``` + +E valide o bucket resolvido por `APP_ENV`/`storage.yaml`. + +### OCI SDK falha em dedicated endpoint + +Confira: + +```env +LLM_PROVIDER=oci_sdk +OCI_GENAI_BASE_URL=https:// +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... +OCI_COMPARTMENT_ID=... +``` + +Para `oci_sdk`, não use `/openai/v1` no endpoint. + +### Banco não inicializado + +```bash +python -m evaluator.cli init-db +``` + +### Ver progresso pela CLI + +O `run-agents` agora imprime o `RUN_ID` assim que a execução é criada: + +```text +[telecom_contas] RUN_ID=8d9a2d1e-... + progress: python -m evaluator.cli progress 8d9a2d1e-... --events 100 +``` + +Também é possível listar runs e consultar qualquer execução: + +```bash +python -m evaluator.cli runs +python -m evaluator.cli progress --events 100 +``` + +### Ver progresso pela interface web + +```bash +python -m uvicorn evaluator_monitor.app:app --host 0.0.0.0 --port 8080 +``` + +Abra `http://localhost:8080/ui`. Em Kubernetes, prefira `kubectl port-forward svc/evaluator-monitor 8080:8080`. + +--- + +## 22. Variáveis de ambiente principais + +| Variável | Uso | +|---|---| +| `APP_ENV` | ambiente usado para resolver storage | +| `ADB_*` | conexão e wallet Oracle | +| `ENABLE_LANGFUSE` | habilita coleta Langfuse | +| `LANGFUSE_CREDENTIALS_CONFIG_PATH` | YAML declarativo por agente | +| `LANGFUSE_*` | fallback global apenas para `run` manual | +| `PUBLISH_LANGFUSE_SCORES` | publica scores de trace de volta no Langfuse | +| `LLM_PROVIDER` | `mock`, `oci_openai`, `oci_sdk`, `openai_compatible` | +| `LLM_PROFILE` | perfil de judge | +| `OCI_AUTH_MODE` | autenticação OCI SDK | +| `OCI_GENAI_BASE_URL` | endpoint OCI GenAI | +| `OCI_GENAI_MODEL` | model ID/nome ou compatibilidade com endpoint OCID | +| `OCI_GENAI_ENDPOINT_ID` | dedicated endpoint OCID | +| `OCI_COMPARTMENT_ID` | compartment OCI | +| `AGENTS_CONFIG_PATH` | arquivo de agentes | +| `TRACE_PROMPT_PATH` | prompt trace-level | +| `SESSION_PROMPT_PATH` | prompt session-level | +| `BATCH_SIZE` | quantidade processada por lote interno | +| `MAX_ATTEMPTS` | limite configurável de tentativas | +| `CONTEXT_TURN_LIMIT` | últimos N turnos enviados ao judge trace | +| `ENABLE_GCS_UPLOAD` | exige upload no fluxo produtivo | +| `STORAGE_CONFIG_PATH` | mapping de buckets por ambiente | +| `JUDGE_GCS_BUCKET` | override de bucket | +| `GOOGLE_APPLICATION_CREDENTIALS` | credencial GCP | +| `OUTPUT_DIR` | diretório temporário de export | + +--- + +## 23. Segurança e operação + +- não versionar `.env` real; +- não versionar `langfuse_credentials.yaml` real; +- não versionar GCP service account JSON; +- não versionar wallet Oracle; +- em OKE prefira `oke_workload_identity` quando aplicável; +- monte credenciais como Kubernetes Secrets; +- use `readOnlyRootFilesystem` no CronJob; +- mantenha `concurrencyPolicy: Forbid` para evitar dois D+1 concorrentes do mesmo deployment; +- nunca use `reset-db` em produção sem plano explícito de restauração. + +--- + +## 24. Checklist de homologação + +Antes de promover uma versão: + +- [ ] `pip check` sem erro; +- [ ] `python -m pytest -v` passando; +- [ ] `python -m compileall evaluator` sem erro; +- [ ] `show-config` aponta para ADB/OCI/configs corretos; +- [ ] `agents.yaml` contém apenas os agentes esperados para aquele deployment; +- [ ] credencial Langfuse existe para cada `agentId` habilitado; +- [ ] `run-agents --limit 5` encontra traces; +- [ ] sampling seleciona sessions, não traces individuais; +- [ ] judge trace/session conclui; +- [ ] arquivo possui cabeçalho de 12 colunas; +- [ ] nome segue `AGENTE_{agentId}_LLM_JUDGE_{YYYYMMDD}_{HHMMSS}.TXT.GZ`; +- [ ] evento `GCS_UPLOADED` confirmado; +- [ ] arquivo local removido depois do upload; +- [ ] auditoria Langfuse `created/completed` confirmada; +- [ ] CronJob está `suspend: false`; +- [ ] timezone do CronJob é `America/Sao_Paulo`; +- [ ] nenhuma credencial real está no repositório. + +--- + +## 25. Resumo para quem está chegando no projeto + +Se você precisa entender o projeto em poucos minutos, pense assim: + +```text +Langfuse guarda as conversas + ↓ +Evaluator escolhe sessions D+1 + ↓ +OCI GenAI julga traces e sessions + ↓ +ADB guarda execução e resultados + ↓ +GCS recebe o arquivo oficial + ↓ +Langfuse registra a auditoria do batch +``` + +Os pontos de extensão ficam em configuração: + +```text +Qual agente? -> configs/judge/agents.yaml +Qual projeto Langfuse?-> langfuse_credentials.yaml +Qual bucket? -> configs/judge/storage.yaml +Qual prompt? -> trace_metrics.yaml / session_metrics.yaml +Qual modelo? -> .env + llm_profiles.yaml +Qual identidade? -> configs/identity.yaml +``` + +A regra arquitetural mais importante é: **o código do evaluator deve permanecer genérico; detalhes de agentes, ambientes, credenciais e storage pertencem à configuração.** diff --git a/configs/.DS_Store b/configs/.DS_Store new file mode 100644 index 0000000..08521bc Binary files /dev/null and b/configs/.DS_Store differ diff --git a/configs/identity.yaml b/configs/identity.yaml new file mode 100644 index 0000000..5f20147 --- /dev/null +++ b/configs/identity.yaml @@ -0,0 +1,55 @@ +identity: + version: "2" + required: + - session_key + keys: + customer_key: + description: Cliente/assinante/consumidor canônico. + sources: + - business_context.customer_key + - customer_key + - msisdn + - customer_id + - user_id + - ani + - from + contract_key: + description: Contrato, conta, fatura, pedido ou asset principal. + sources: + - business_context.contract_key + - contract_key + - invoice_id + - current_invoice_number + - order_id + - pedido_id + - asset_id + interaction_key: + description: Chave externa da interação/call/chat vinda do canal. + sources: + - business_context.interaction_key + - interaction_key + - ura_call_id + - call_id + - message_id + account_key: + description: Conta de cobrança/conta comercial. + sources: + - business_context.account_key + - account_key + - account_id + - billing_account_id + resource_key: + description: Recurso/linha/produto/asset específico. + sources: + - business_context.resource_key + - resource_key + - asset_id + - product_id + - sku + session_key: + description: Sessão técnica estável já escopada por tenant e agente. + sources: + - business_context.session_key + - session_key + - conversation_key + - session_id diff --git a/configs/judge/agents.yaml b/configs/judge/agents.yaml new file mode 100644 index 0000000..e33eb86 --- /dev/null +++ b/configs/judge/agents.yaml @@ -0,0 +1,28 @@ +agents: + - agentId: billing + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: [] + gcs_prefix: agnt_ai_contas + + - agentId: sales + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: [] + gcs_prefix: agnt_ai_ofertas + + - agentId: collection + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: [] + gcs_prefix: agnt_ai_cobranca + + - agentId: backoffice-anatel + enabled: true + daysBack: 1 + percentage: 1.0 + langfuse_agent_aliases: [] + gcs_prefix: agnt_ai_bko diff --git a/configs/judge/session_metrics.yaml b/configs/judge/session_metrics.yaml new file mode 100644 index 0000000..44a4831 --- /dev/null +++ b/configs/judge/session_metrics.yaml @@ -0,0 +1,17 @@ +session_metrics: | + Você é um avaliador imparcial de uma sessão completa de conversa entre + cliente e agente. Vai receber a TRANSCRIÇÃO da sessão (alternância + user/agent). Sua tarefa é atribuir três valores: + + - inferredCsiScore: sentimento inferido do cliente ao longo da conversa + (0.0 negativo, 0.5 neutro, 1.0 positivo). + - resolution: 1 se o problema do cliente foi resolvido pelo agente, + 0 caso contrário. + - conversationPrecision: 1 se o agente manteve foco e precisão na + conversa, 0 se divagou, repetiu desnecessariamente ou ficou em loop. + + Retorne SOMENTE um JSON válido, sem texto adicional: + + {"inferredCsiScore": , "resolution": <0|1>, "conversationPrecision": <0|1>, "rationale": ""} + + rationale deve ter no máximo 200 caracteres. diff --git a/configs/judge/storage.yaml b/configs/judge/storage.yaml new file mode 100644 index 0000000..6893939 --- /dev/null +++ b/configs/judge/storage.yaml @@ -0,0 +1,5 @@ +storage: + buckets: + dev: tim-bigdata-dev-ca1f-raw-standard-agnt-ai-atendimento + fqa: tim-bigdata-fqa-60ce-raw-standard-agnt-ai-atendimento + prd: tim-bigdata-prod-e305-raw-standard-agnt-ai-atendimento diff --git a/configs/judge/trace_metrics.yaml b/configs/judge/trace_metrics.yaml new file mode 100644 index 0000000..6e73b5a --- /dev/null +++ b/configs/judge/trace_metrics.yaml @@ -0,0 +1,16 @@ +trace_metrics: | + Você é um avaliador imparcial de respostas de agentes conversacionais. + Vai receber o HISTÓRICO da conversa (últimos turnos), a MENSAGEM DO + USUÁRIO e a RESPOSTA DO AGENTE. Sua tarefa é atribuir três notas + numéricas entre 0.0 e 1.0: + + - judgeScore: qualidade global da resposta (clareza, utilidade, tom). + - accuracyScore: factualidade da resposta frente ao contexto fornecido. + - alucinationScore: grau em que a resposta contém afirmações NÃO + suportadas pelo contexto (alto = mais alucinação = pior). + + Retorne SOMENTE um JSON válido, sem nenhum texto adicional, no formato: + + {"judgeScore": , "accuracyScore": , "alucinationScore": , "rationale": ""} + + rationale deve ter no máximo 200 caracteres. diff --git a/configs/langfuse/langfuse_credentials.example.yaml b/configs/langfuse/langfuse_credentials.example.yaml new file mode 100644 index 0000000..3b061d9 --- /dev/null +++ b/configs/langfuse/langfuse_credentials.example.yaml @@ -0,0 +1,22 @@ +# Copy/mount this file as configs/langfuse/langfuse_credentials.yaml, or point +# LANGFUSE_CREDENTIALS_CONFIG_PATH to another location. +# +# Production recommendation: mount the complete file from a Kubernetes Secret. +# The evaluator loops over all entries; adding an agent requires no Python change. +langfuse: + agents: + - agentId: billing + host: https://langfuse.example.internal + publicKey: pk-example-billing + secretKey: sk-example-billing + + - agentId: sales + host: https://langfuse.example.internal + publicKey: pk-example-sales + secretKey: sk-example-sales + +# Alternative: keep only references in Git and source values from env or files: +# - agentId: collection +# hostEnv: COLLECTION_LANGFUSE_HOST +# publicKeyEnv: COLLECTION_LANGFUSE_PUBLIC_KEY +# secretKeyFile: /var/secrets/langfuse/collection-secret-key diff --git a/configs/llm_profiles/llm_profiles.yaml b/configs/llm_profiles/llm_profiles.yaml new file mode 100644 index 0000000..3caa69b --- /dev/null +++ b/configs/llm_profiles/llm_profiles.yaml @@ -0,0 +1,9 @@ +profiles: + judge: + temperature: 0 + max_tokens: 900 + mock: + provider: mock + model: mock-judge + temperature: 0 + max_tokens: 300 diff --git a/deploy/.DS_Store b/deploy/.DS_Store new file mode 100644 index 0000000..01a15a8 Binary files /dev/null and b/deploy/.DS_Store differ diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..c5836fc --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,125 @@ +# Deployment Kubernetes — Evaluator TIM + +O **evaluator continua sendo batch/CronJob-only**. O componente `evaluator-batch` não expõe HTTP nem gRPC. + +Para acompanhamento operacional existe um componente **separado e somente leitura**, `evaluator-monitor`, que consulta as tabelas do Autonomous Database e disponibiliza `/ui` e APIs de monitoramento. Essa separação mantém o processamento em batch e evita acoplar uma API HTTP ao evaluator. + +Recursos aplicados pelo Kustomize: + +```text +Namespace +ServiceAccount +ConfigMap +CronJob evaluator-batch +Deployment evaluator-monitor +Service evaluator-monitor (ClusterIP) +``` + +O CronJob executa diariamente às 02:00 em `America/Sao_Paulo`, com `suspend: false` e `concurrencyPolicy: Forbid`. + +## Preparação + +```bash +cp deploy/env/evaluator.env.example deploy/env/evaluator.env +``` + +Configure ADB, OCI GenAI e GCS no env file. Copie `deploy/langfuse/langfuse_credentials.example.yaml` para `deploy/langfuse/langfuse_credentials.yaml` e declare ali quantos agentes/projetos Langfuse forem necessários. O script cria o Secret `evaluator-langfuse` dinamicamente a partir desse arquivo. + +Wallet Oracle: + +```text +deploy/wallet/ +``` + +Credencial GCP opcionalmente materializada pelo script: + +```text +deploy/gcp/GCP_ACCESS_KEY.json +``` + +## Secrets + +```bash +./deploy/scripts/create-secrets.sh dev +./deploy/scripts/create-secrets.sh hml +./deploy/scripts/create-secrets.sh prod +``` + +## Build + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG=0.3.0 +./deploy/scripts/build-push.sh prod +``` + +## Deploy + +```bash +export REGISTRY=tim.azurecr.io +export IMAGE_TAG=0.3.0 +./deploy/scripts/deploy.sh prod +``` + +## Status + +```bash +./deploy/scripts/status.sh prod +``` + +Ou: + +```bash +kubectl -n agent-evaluator-prod get cronjob,jobs,pods,deploy,svc +``` + +## Execução imediata + +```bash +./deploy/scripts/run-now.sh prod +``` + +Acompanhar logs do batch: + +```bash +kubectl -n agent-evaluator-prod get jobs,pods +kubectl -n agent-evaluator-prod logs -f job/ +``` + +O comando `run-agents` também imprime o `RUN_ID` assim que cada execução é criada, permitindo: + +```bash +python -m evaluator.cli progress --events 100 +``` + +## Interface web + +O Service do monitor é `ClusterIP`, portanto não fica exposto externamente por default. + +Para acessar via sua máquina: + +```bash +kubectl -n agent-evaluator-prod port-forward svc/evaluator-monitor 8080:8080 +``` + +Abra: + +```text +http://localhost:8080/ui +``` + +Endpoints read-only: + +```text +GET /health +GET /api/runs +GET /api/runs/{run_id} +GET /api/runs/{run_id}/results +GET /ui +``` + +Não há endpoints para iniciar, cancelar, apagar ou alterar runs. + +## Configuração de storage + +Os buckets por ambiente ficam em `configs/judge/storage.yaml`. Para usar outro arquivo, configure `STORAGE_CONFIG_PATH`. `JUDGE_GCS_BUCKET` sobrescreve o mapping quando necessário. diff --git a/deploy/env/evaluator.env.example b/deploy/env/evaluator.env.example new file mode 100644 index 0000000..4d80ed3 --- /dev/null +++ b/deploy/env/evaluator.env.example @@ -0,0 +1,22 @@ +# Copie para evaluator.env e não faça commit do arquivo real. +APP_ENV=dev +ADB_USER=admin +ADB_PASSWORD=change-me +ADB_DSN=oradb23ai_high +ADB_WALLET_PASSWORD=change-me + +LLM_PROVIDER=oci_sdk +OCI_AUTH_MODE=oke_workload_identity +OCI_GENAI_BASE_URL=https://.inference.generativeai.sa-saopaulo-1.oci.oraclecloud.com +OCI_GENAI_MODEL=ocid1.generativeaiendpoint... +OCI_GENAI_ENDPOINT_ID=ocid1.generativeaiendpoint... +OCI_COMPARTMENT_ID=ocid1.compartment... +OCI_REGION=sa-saopaulo-1 + +ENABLE_GCS_UPLOAD=true +JUDGE_GCS_BUCKET= +STORAGE_CONFIG_PATH=configs/judge/storage.yaml +GOOGLE_APPLICATION_CREDENTIALS=/var/secrets/google/GCP_ACCESS_KEY.json + +# Langfuse per-agent credentials are supplied by the mounted YAML file, not env names. +LANGFUSE_CREDENTIALS_CONFIG_PATH=/var/secrets/langfuse/langfuse_credentials.yaml diff --git a/deploy/k8s/.DS_Store b/deploy/k8s/.DS_Store new file mode 100644 index 0000000..707cc98 Binary files /dev/null and b/deploy/k8s/.DS_Store differ diff --git a/deploy/k8s/base/configmap.yaml b/deploy/k8s/base/configmap.yaml new file mode 100644 index 0000000..22edd21 --- /dev/null +++ b/deploy/k8s/base/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: evaluator-config + namespace: agent-evaluator +data: + ENABLE_LANGFUSE: "true" + LANGFUSE_CREDENTIALS_CONFIG_PATH: "/var/secrets/langfuse/langfuse_credentials.yaml" + PUBLISH_LANGFUSE_SCORES: "false" + LLM_PROVIDER: "oci_sdk" + LLM_PROFILE: "judge" + LLM_PROFILES_PATH: "configs/llm_profiles/llm_profiles.yaml" + LLM_TEMPERATURE: "0" + LLM_MAX_TOKENS: "900" + LLM_TIMEOUT_SECONDS: "120" + AGENTS_CONFIG_PATH: "configs/judge/agents.yaml" + TRACE_PROMPT_PATH: "configs/judge/trace_metrics.yaml" + SESSION_PROMPT_PATH: "configs/judge/session_metrics.yaml" + OUTPUT_DIR: "/app/output" + BATCH_SIZE: "50" + MAX_ATTEMPTS: "3" + ENABLE_GCS_UPLOAD: "true" + STORAGE_CONFIG_PATH: "configs/judge/storage.yaml" + ADB_TABLE_PREFIX: "AGENTFW" + ADB_WALLET_LOCATION: "/opt/oracle/wallet" diff --git a/deploy/k8s/base/cronjob.yaml b/deploy/k8s/base/cronjob.yaml new file mode 100644 index 0000000..44e3caf --- /dev/null +++ b/deploy/k8s/base/cronjob.yaml @@ -0,0 +1,94 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: evaluator-batch + namespace: agent-evaluator + labels: + app.kubernetes.io/name: evaluator-batch + app.kubernetes.io/component: batch + app.kubernetes.io/part-of: agent-framework +spec: + schedule: "0 2 * * *" + timeZone: "America/Sao_Paulo" + suspend: false + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + startingDeadlineSeconds: 1800 + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 21600 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + app.kubernetes.io/name: evaluator-batch + app.kubernetes.io/component: batch + spec: + restartPolicy: Never + serviceAccountName: evaluator + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: evaluator + image: agent-framework-evaluator:latest + imagePullPolicy: IfNotPresent + command: ["python", "-m", "evaluator.cli", "run-agents"] + args: ["--source", "langfuse"] + envFrom: + - configMapRef: + name: evaluator-config + - secretRef: + name: evaluator-secrets + env: + - name: TNS_ADMIN + value: /opt/oracle/wallet + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: wallet + mountPath: /opt/oracle/wallet + readOnly: true + - name: output + mountPath: /app/output + - name: tmp + mountPath: /tmp + - name: gcp-credentials + mountPath: /var/secrets/google + readOnly: true + - name: langfuse-credentials + mountPath: /var/secrets/langfuse + readOnly: true + volumes: + - name: wallet + secret: + secretName: evaluator-wallet + - name: output + emptyDir: {} + - name: tmp + emptyDir: {} + - name: gcp-credentials + secret: + secretName: evaluator-gcp + optional: true + - name: langfuse-credentials + secret: + secretName: evaluator-langfuse + optional: false diff --git a/deploy/k8s/base/init-db-job.yaml b/deploy/k8s/base/init-db-job.yaml new file mode 100644 index 0000000..98a8255 --- /dev/null +++ b/deploy/k8s/base/init-db-job.yaml @@ -0,0 +1,63 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: evaluator-init-db + namespace: agent-evaluator + labels: + app.kubernetes.io/name: evaluator-init-db + app.kubernetes.io/component: database-migration +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: evaluator-init-db + spec: + restartPolicy: OnFailure + serviceAccountName: evaluator + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: init-db + image: agent-framework-evaluator:latest + imagePullPolicy: IfNotPresent + command: ["python", "-m", "evaluator.cli", "init-db"] + envFrom: + - configMapRef: + name: evaluator-config + - secretRef: + name: evaluator-secrets + env: + - name: TNS_ADMIN + value: /opt/oracle/wallet + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: wallet + mountPath: /opt/oracle/wallet + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: wallet + secret: + secretName: evaluator-wallet + - name: tmp + emptyDir: {} diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml new file mode 100644 index 0000000..342631b --- /dev/null +++ b/deploy/k8s/base/kustomization.yaml @@ -0,0 +1,12 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agent-evaluator +resources: + - namespace.yaml + - serviceaccount.yaml + - configmap.yaml + - cronjob.yaml + - monitor-deployment.yaml + - monitor-service.yaml +commonLabels: + app.kubernetes.io/managed-by: kustomize diff --git a/deploy/k8s/base/monitor-deployment.yaml b/deploy/k8s/base/monitor-deployment.yaml new file mode 100644 index 0000000..046a97f --- /dev/null +++ b/deploy/k8s/base/monitor-deployment.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: evaluator-monitor + namespace: agent-evaluator + labels: + app.kubernetes.io/name: evaluator-monitor + app.kubernetes.io/component: monitoring + app.kubernetes.io/part-of: agent-framework +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: evaluator-monitor + template: + metadata: + labels: + app.kubernetes.io/name: evaluator-monitor + app.kubernetes.io/component: monitoring + spec: + serviceAccountName: evaluator + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: monitor + image: agent-framework-evaluator:latest + imagePullPolicy: IfNotPresent + command: ["python", "-m", "uvicorn"] + args: ["evaluator_monitor.app:app", "--host", "0.0.0.0", "--port", "8080"] + ports: + - name: http + containerPort: 8080 + envFrom: + - configMapRef: + name: evaluator-config + - secretRef: + name: evaluator-secrets + env: + - name: TNS_ADMIN + value: /opt/oracle/wallet + readinessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 15 + periodSeconds: 20 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: wallet + mountPath: /opt/oracle/wallet + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: wallet + secret: + secretName: evaluator-wallet + - name: tmp + emptyDir: {} diff --git a/deploy/k8s/base/monitor-service.yaml b/deploy/k8s/base/monitor-service.yaml new file mode 100644 index 0000000..138e88e --- /dev/null +++ b/deploy/k8s/base/monitor-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: evaluator-monitor + namespace: agent-evaluator + labels: + app.kubernetes.io/name: evaluator-monitor + app.kubernetes.io/component: monitoring +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: evaluator-monitor + ports: + - name: http + port: 8080 + targetPort: http diff --git a/deploy/k8s/base/namespace.yaml b/deploy/k8s/base/namespace.yaml new file mode 100644 index 0000000..dd2b322 --- /dev/null +++ b/deploy/k8s/base/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: agent-evaluator + labels: + app.kubernetes.io/part-of: agent-framework diff --git a/deploy/k8s/base/secret-template.yaml b/deploy/k8s/base/secret-template.yaml new file mode 100644 index 0000000..93e3bf9 --- /dev/null +++ b/deploy/k8s/base/secret-template.yaml @@ -0,0 +1,35 @@ +# TEMPLATE ONLY. Do not apply or commit real values. +apiVersion: v1 +kind: Secret +metadata: + name: evaluator-secrets + namespace: agent-evaluator +type: Opaque +stringData: + ADB_USER: "REPLACE_ME" + ADB_PASSWORD: "REPLACE_ME" + ADB_DSN: "REPLACE_ME" + ADB_WALLET_PASSWORD: "REPLACE_ME" + OCI_GENAI_BASE_URL: "https://private-endpoint.inference.generativeai.region.oci.oraclecloud.com" + OCI_GENAI_MODEL: "ocid1.generativeaiendpoint.REPLACE_ME" + OCI_GENAI_ENDPOINT_ID: "ocid1.generativeaiendpoint.REPLACE_ME" + OCI_COMPARTMENT_ID: "ocid1.compartment.REPLACE_ME" + OCI_AUTH_MODE: "oke_workload_identity" + GOOGLE_APPLICATION_CREDENTIALS: "/var/secrets/google/GCP_ACCESS_KEY.json" +--- +# Langfuse credentials are one data-driven YAML, not one env namespace per agent. +# Add as many entries as required; no Python or manifest change is needed. +apiVersion: v1 +kind: Secret +metadata: + name: evaluator-langfuse + namespace: agent-evaluator +type: Opaque +stringData: + langfuse_credentials.yaml: | + langfuse: + agents: + - agentId: REPLACE_WITH_AGENT_ID + host: https://langfuse.example + publicKey: pk-REPLACE_ME + secretKey: sk-REPLACE_ME diff --git a/deploy/k8s/base/serviceaccount.yaml b/deploy/k8s/base/serviceaccount.yaml new file mode 100644 index 0000000..d0ec71e --- /dev/null +++ b/deploy/k8s/base/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: evaluator + namespace: agent-evaluator +automountServiceAccountToken: false diff --git a/deploy/k8s/overlays/dev/kustomization.yaml b/deploy/k8s/overlays/dev/kustomization.yaml new file mode 100644 index 0000000..888b487 --- /dev/null +++ b/deploy/k8s/overlays/dev/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agent-evaluator-dev +resources: + - ../../base +images: + - name: agent-framework-evaluator + newName: REPLACE_REGISTRY/agent-framework-evaluator + newTag: dev +patches: + - target: + kind: Namespace + name: agent-evaluator + patch: |- + - op: replace + path: /metadata/name + value: agent-evaluator-dev + - target: + kind: CronJob + name: evaluator-batch + patch: |- + - op: replace + path: /spec/suspend + value: false diff --git a/deploy/k8s/overlays/hml/kustomization.yaml b/deploy/k8s/overlays/hml/kustomization.yaml new file mode 100644 index 0000000..8821b3b --- /dev/null +++ b/deploy/k8s/overlays/hml/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agent-evaluator-hml +resources: + - ../../base +images: + - name: agent-framework-evaluator + newName: REPLACE_REGISTRY/agent-framework-evaluator + newTag: hml +patches: + - target: + kind: Namespace + name: agent-evaluator + patch: |- + - op: replace + path: /metadata/name + value: agent-evaluator-hml + - target: + kind: CronJob + name: evaluator-batch + patch: |- + - op: replace + path: /spec/suspend + value: false diff --git a/deploy/k8s/overlays/prod/kustomization.yaml b/deploy/k8s/overlays/prod/kustomization.yaml new file mode 100644 index 0000000..5949df6 --- /dev/null +++ b/deploy/k8s/overlays/prod/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agent-evaluator-prod +resources: + - ../../base +images: + - name: agent-framework-evaluator + newName: REPLACE_REGISTRY/agent-framework-evaluator + newTag: stable +patches: + - target: + kind: Namespace + name: agent-evaluator + patch: |- + - op: replace + path: /metadata/name + value: agent-evaluator-prod + - target: + kind: CronJob + name: evaluator-batch + patch: |- + - op: replace + path: /spec/suspend + value: false diff --git a/deploy/langfuse/langfuse_credentials.example.yaml b/deploy/langfuse/langfuse_credentials.example.yaml new file mode 100644 index 0000000..3b061d9 --- /dev/null +++ b/deploy/langfuse/langfuse_credentials.example.yaml @@ -0,0 +1,22 @@ +# Copy/mount this file as configs/langfuse/langfuse_credentials.yaml, or point +# LANGFUSE_CREDENTIALS_CONFIG_PATH to another location. +# +# Production recommendation: mount the complete file from a Kubernetes Secret. +# The evaluator loops over all entries; adding an agent requires no Python change. +langfuse: + agents: + - agentId: billing + host: https://langfuse.example.internal + publicKey: pk-example-billing + secretKey: sk-example-billing + + - agentId: sales + host: https://langfuse.example.internal + publicKey: pk-example-sales + secretKey: sk-example-sales + +# Alternative: keep only references in Git and source values from env or files: +# - agentId: collection +# hostEnv: COLLECTION_LANGFUSE_HOST +# publicKeyEnv: COLLECTION_LANGFUSE_PUBLIC_KEY +# secretKeyFile: /var/secrets/langfuse/collection-secret-key diff --git a/deploy/scripts/build-push.sh b/deploy/scripts/build-push.sh new file mode 100644 index 0000000..8e46159 --- /dev/null +++ b/deploy/scripts/build-push.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command docker +validate_environment + +REGISTRY="${REGISTRY:?Defina REGISTRY, por exemplo: tim.azurecr.io}" +IMAGE_NAME="${IMAGE_NAME:-agent-framework-evaluator}" +IMAGE_TAG="${IMAGE_TAG:-$(git -C "${ROOT_DIR}" rev-parse --short HEAD 2>/dev/null || date +%Y%m%d%H%M%S)}" +IMAGE="${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}" +PLATFORM="${PLATFORM:-linux/amd64}" + +echo "Construindo ${IMAGE} para ${PLATFORM}..." +docker build --platform "${PLATFORM}" -t "${IMAGE}" "${ROOT_DIR}" + +echo "Enviando ${IMAGE}..." +docker push "${IMAGE}" +echo "IMAGE=${IMAGE}" diff --git a/deploy/scripts/common.sh b/deploy/scripts/common.sh new file mode 100644 index 0000000..ad4f834 --- /dev/null +++ b/deploy/scripts/common.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +ENVIRONMENT="${ENVIRONMENT:-${1:-dev}}" +NAMESPACE="${NAMESPACE:-agent-evaluator-${ENVIRONMENT}}" +OVERLAY_DIR="${ROOT_DIR}/deploy/k8s/overlays/${ENVIRONMENT}" + +require_command() { + command -v "$1" >/dev/null 2>&1 || { echo "Erro: comando '$1' não encontrado." >&2; exit 1; } +} + +validate_environment() { + case "${ENVIRONMENT}" in dev|hml|prod) ;; *) echo "Ambiente inválido: ${ENVIRONMENT}. Use dev, hml ou prod." >&2; exit 1;; esac + [[ -d "${OVERLAY_DIR}" ]] || { echo "Overlay não encontrado: ${OVERLAY_DIR}" >&2; exit 1; } +} diff --git a/deploy/scripts/create-secrets.sh b/deploy/scripts/create-secrets.sh new file mode 100644 index 0000000..decb726 --- /dev/null +++ b/deploy/scripts/create-secrets.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command kubectl +validate_environment + +ENV_FILE="${ENV_FILE:-${ROOT_DIR}/deploy/env/evaluator.env}" +WALLET_DIR="${WALLET_DIR:-${ROOT_DIR}/deploy/wallet}" + +[[ -f "${ENV_FILE}" ]] || { echo "Arquivo de variáveis não encontrado: ${ENV_FILE}" >&2; exit 1; } +[[ -d "${WALLET_DIR}" ]] || { echo "Diretório do wallet não encontrado: ${WALLET_DIR}" >&2; exit 1; } + +kubectl create namespace "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - +kubectl -n "${NAMESPACE}" create secret generic evaluator-secrets \ + --from-env-file="${ENV_FILE}" \ + --dry-run=client -o yaml | kubectl apply -f - + +wallet_args=() +while IFS= read -r -d '' file; do + wallet_args+=("--from-file=$(basename "${file}")=${file}") +done < <(find "${WALLET_DIR}" -maxdepth 1 -type f -print0) + +((${#wallet_args[@]} > 0)) || { echo "Nenhum arquivo encontrado em ${WALLET_DIR}" >&2; exit 1; } +kubectl -n "${NAMESPACE}" create secret generic evaluator-wallet \ + "${wallet_args[@]}" \ + --dry-run=client -o yaml | kubectl apply -f - + + + +LANGFUSE_CREDENTIALS_FILE="${LANGFUSE_CREDENTIALS_FILE:-${ROOT_DIR}/deploy/langfuse/langfuse_credentials.yaml}" +if [[ -f "${LANGFUSE_CREDENTIALS_FILE}" ]]; then + kubectl -n "${NAMESPACE}" create secret generic evaluator-langfuse \ + --from-file=langfuse_credentials.yaml="${LANGFUSE_CREDENTIALS_FILE}" \ + --dry-run=client -o yaml | kubectl apply -f - + echo "Credenciais Langfuse atualizadas no secret evaluator-langfuse." +else + echo "Arquivo de credenciais Langfuse não encontrado: ${LANGFUSE_CREDENTIALS_FILE}" >&2 + exit 1 +fi + +GCP_CREDENTIALS_FILE="${GCP_CREDENTIALS_FILE:-${ROOT_DIR}/deploy/gcp/GCP_ACCESS_KEY.json}" +if [[ -f "${GCP_CREDENTIALS_FILE}" ]]; then + kubectl -n "${NAMESPACE}" create secret generic evaluator-gcp \ + --from-file=GCP_ACCESS_KEY.json="${GCP_CREDENTIALS_FILE}" \ + --dry-run=client -o yaml | kubectl apply -f - + echo "Credencial GCP atualizada no secret evaluator-gcp." +else + echo "Aviso: credencial GCP não encontrada em ${GCP_CREDENTIALS_FILE}; secret evaluator-gcp não foi alterado." >&2 +fi + +echo "Secrets atualizados no namespace ${NAMESPACE}." diff --git a/deploy/scripts/deploy.sh b/deploy/scripts/deploy.sh new file mode 100644 index 0000000..07bf9f5 --- /dev/null +++ b/deploy/scripts/deploy.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command kubectl +validate_environment + +REGISTRY="${REGISTRY:?Defina REGISTRY, por exemplo: tim.azurecr.io}" +IMAGE_NAME="${IMAGE_NAME:-agent-framework-evaluator}" +IMAGE_TAG="${IMAGE_TAG:?Defina IMAGE_TAG com a versão já publicada}" +IMAGE="${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}" +RUN_INIT_DB="${RUN_INIT_DB:-true}" +TIMEOUT="${TIMEOUT:-300s}" + +kubectl kustomize "${OVERLAY_DIR}" \ + | sed "s#REPLACE_REGISTRY/agent-framework-evaluator:[^[:space:]]*#${IMAGE}#g" \ + | kubectl apply -f - + +if [[ "${RUN_INIT_DB}" == "true" ]]; then + job_name="evaluator-init-db-${ENVIRONMENT}-$(date +%s)" + sed -e "s#agent-evaluator#${NAMESPACE}#g" \ + -e "s#name: evaluator-init-db#name: ${job_name}#" \ + -e "s#agent-framework-evaluator:latest#${IMAGE}#" \ + "${ROOT_DIR}/deploy/k8s/base/init-db-job.yaml" | kubectl apply -f - + kubectl -n "${NAMESPACE}" wait --for=condition=complete "job/${job_name}" --timeout="${TIMEOUT}" || { + kubectl -n "${NAMESPACE}" logs "job/${job_name}" --all-containers=true || true + exit 1 + } +fi + +kubectl -n "${NAMESPACE}" get cronjob,pod,jobs diff --git a/deploy/scripts/run-now.sh b/deploy/scripts/run-now.sh new file mode 100644 index 0000000..36c0c60 --- /dev/null +++ b/deploy/scripts/run-now.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command kubectl +validate_environment + +CRONJOB="${CRONJOB:-evaluator-batch}" +JOB_NAME="${JOB_NAME:-evaluator-manual-${ENVIRONMENT}-$(date +%Y%m%d%H%M%S)}" +kubectl -n "${NAMESPACE}" create job --from="cronjob/${CRONJOB}" "${JOB_NAME}" +echo "Job criado: ${JOB_NAME}" +echo "Logs: kubectl -n ${NAMESPACE} logs -f job/${JOB_NAME}" diff --git a/deploy/scripts/status.sh b/deploy/scripts/status.sh new file mode 100644 index 0000000..fa3bba8 --- /dev/null +++ b/deploy/scripts/status.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command kubectl +validate_environment +kubectl -n "${NAMESPACE}" get cronjob,jobs,pods,deploy,svc +printf '\nEventos recentes:\n' +kubectl -n "${NAMESPACE}" get events --sort-by=.lastTimestamp | tail -30 diff --git a/deploy/scripts/undeploy.sh b/deploy/scripts/undeploy.sh new file mode 100644 index 0000000..5a70183 --- /dev/null +++ b/deploy/scripts/undeploy.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +source "$(dirname "$0")/common.sh" +require_command kubectl +validate_environment + +kubectl delete -k "${OVERLAY_DIR}" --ignore-not-found +if [[ "${DELETE_NAMESPACE:-false}" == "true" ]]; then + kubectl delete namespace "${NAMESPACE}" --ignore-not-found +fi diff --git a/evaluator/.DS_Store b/evaluator/.DS_Store new file mode 100644 index 0000000..fe416d7 Binary files /dev/null and b/evaluator/.DS_Store differ diff --git a/evaluator/__init__.py b/evaluator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evaluator/__pycache__/__init__.cpython-313.pyc b/evaluator/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..8c9c51a Binary files /dev/null and b/evaluator/__pycache__/__init__.cpython-313.pyc differ diff --git a/evaluator/__pycache__/cli.cpython-313.pyc b/evaluator/__pycache__/cli.cpython-313.pyc new file mode 100644 index 0000000..84070a3 Binary files /dev/null and b/evaluator/__pycache__/cli.cpython-313.pyc differ diff --git a/evaluator/__pycache__/engine.cpython-313.pyc b/evaluator/__pycache__/engine.cpython-313.pyc new file mode 100644 index 0000000..388cdb7 Binary files /dev/null and b/evaluator/__pycache__/engine.cpython-313.pyc differ diff --git a/evaluator/__pycache__/time_utils.cpython-313.pyc b/evaluator/__pycache__/time_utils.cpython-313.pyc new file mode 100644 index 0000000..57df6f4 Binary files /dev/null and b/evaluator/__pycache__/time_utils.cpython-313.pyc differ diff --git a/evaluator/analytics/__init__.py b/evaluator/analytics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evaluator/analytics/__pycache__/__init__.cpython-313.pyc b/evaluator/analytics/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..d766580 Binary files /dev/null and b/evaluator/analytics/__pycache__/__init__.cpython-313.pyc differ diff --git a/evaluator/analytics/__pycache__/vloop.cpython-313.pyc b/evaluator/analytics/__pycache__/vloop.cpython-313.pyc new file mode 100644 index 0000000..5edba9e Binary files /dev/null and b/evaluator/analytics/__pycache__/vloop.cpython-313.pyc differ diff --git a/evaluator/analytics/vloop.py b/evaluator/analytics/vloop.py new file mode 100644 index 0000000..f2c05af --- /dev/null +++ b/evaluator/analytics/vloop.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from collections import defaultdict +from typing import Any, Iterable + + +def _as_dict(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def extract_intent(record_or_raw: Any) -> str | None: + """Extract the trace intent used by the TIM specification for the deterministic loop metric.""" + if isinstance(record_or_raw, dict): + metadata = _as_dict(record_or_raw.get("metadata")) + nested_raw = _as_dict(record_or_raw.get("raw")) + else: + metadata = _as_dict(getattr(record_or_raw, "metadata", None)) + nested_raw = _as_dict(getattr(record_or_raw, "raw", None)) + + trace = _as_dict(nested_raw.get("trace")) + detail = _as_dict(nested_raw.get("detail")) + trace_meta = _as_dict(trace.get("metadata")) + detail_meta = _as_dict(detail.get("metadata")) + + value = ( + metadata.get("intent") + or trace_meta.get("intent") + or detail_meta.get("intent") + ) + if value is None: + return None + text = str(value).strip() + return text or None + + +def max_consecutive_intent_streak(records: Iterable[Any]) -> int: + """Maximum consecutive streak of the same metadata.intent in trace order.""" + maximum = 0 + current = 0 + previous: str | None = None + + for record in records: + intent = extract_intent(record) + if not intent: + previous = None + current = 0 + continue + if intent == previous: + current += 1 + else: + previous = intent + current = 1 + maximum = max(maximum, current) + + return maximum + + +def session_loop_values(records: Iterable[Any], session_id_getter=None) -> dict[str, int]: + grouped: dict[str, list[Any]] = defaultdict(list) + for record in records: + if session_id_getter: + sid = session_id_getter(record) + elif isinstance(record, dict): + sid = record.get("session_id") or record.get("sessionId") + else: + sid = getattr(record, "session_id", None) + if sid: + grouped[str(sid)].append(record) + return {sid: max_consecutive_intent_streak(items) for sid, items in grouped.items()} + + +# Backward-compatible name used by older integrations. For a single trace the +# streak is either 0 (no intent) or 1 (intent present); the official metric is +# session-level and should use max_consecutive_intent_streak over all traces. +def vloop_flag(record_or_raw: Any) -> int: + return max_consecutive_intent_streak([record_or_raw]) diff --git a/evaluator/cli.py b/evaluator/cli.py new file mode 100644 index 0000000..8544e38 --- /dev/null +++ b/evaluator/cli.py @@ -0,0 +1,93 @@ +from __future__ import annotations +import asyncio +from datetime import datetime, timedelta +import typer +from zoneinfo import ZoneInfo +from rich import print +from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeElapsedColumn +from evaluator.config.agents import load_agents +from evaluator.engine import EvaluationEngine +from evaluator.persistence.repository import EvaluationRepository +from evaluator.config.settings import settings +from evaluator.time_utils import brt_day_window + +app = typer.Typer(help='Agent Framework TIM-style LLM Judge Evaluator') + + +def _run_progress(coro_factory): + async def runner(): + state={'run_id': None} + with Progress(SpinnerColumn(), TextColumn('[bold blue]{task.fields[stage]}'), BarColumn(), TextColumn('{task.completed}/{task.total}'), TextColumn('{task.percentage:>3.0f}%'), TimeElapsedColumn()) as progress: + task=progress.add_task('evaluation', total=1, stage='starting') + async def cb(event): + state['run_id'] = event.get('run_id') or state['run_id'] + stage = event.get('stage','') + msg = event.get('message','') + if state['run_id']: + snap = await EvaluationRepository(auto_init_schema=False).aget_run_progress(state['run_id'], event_limit=1) + total=int(snap.get('total_items') or 0) or 1 + done=int(snap.get('done_items') or 0) + progress.update(task,total=total,completed=done,stage=f'{stage}: {msg}'[:120]) + result = await coro_factory(cb) + progress.update(task, completed=1, total=1, stage='finished') + return result + return asyncio.run(runner()) + +@app.command("reset-db") +def reset_db(): + repo = EvaluationRepository(auto_init_schema=False) + repo.store.drop_schema() + repo.store._init_schema() + print({"status": "OK", "message": "Evaluator schema dropped and recreated successfully."}) + +@app.command('init-db') +def init_db(): + EvaluationRepository(auto_init_schema=True) + print({'status':'OK','message':'schema checked/created'}) + +@app.command('show-config') +def show_config(): + print({'env_path': str(settings.project_root / '.env'), 'adb_dsn': settings.ADB_DSN, 'wallet': settings.ADB_WALLET_LOCATION, 'langfuse': settings.enable_langfuse, 'publish_langfuse_scores': settings.publish_langfuse_scores, 'llm_provider': settings.llm_provider, 'llm_profile': settings.llm_profile, 'oci_genai_base_url': settings.OCI_GENAI_BASE_URL, 'oci_genai_model': settings.OCI_GENAI_MODEL, 'oci_genai_api_key_configured': bool(settings.OCI_GENAI_API_KEY), 'agents_config': settings.agents_config_path}) + +@app.command('run') +def run(period_start: datetime, period_end: datetime, source: str='langfuse', limit: int|None=None, show_progress: bool=True): + if show_progress: + result = _run_progress(lambda cb: EvaluationEngine(progress_callback=cb).run(period_start, period_end, source, limit)) + else: + result = asyncio.run(EvaluationEngine().run(period_start, period_end, source, limit)) + print(result) + +@app.command('run-agents') +def run_agents(source: str='langfuse', agent_id: str|None=None, limit: int|None=None): + async def main(): + results=[] + today_brt = datetime.now(ZoneInfo("America/Sao_Paulo")).date() + for agent in load_agents(): + if agent_id and agent.agent_id != agent_id: + continue + reference_date = today_brt - timedelta(days=agent.days_back) + start, end = brt_day_window(reference_date) + announced = {"run_id": None} + async def announce_run(event, _agent_id=agent.agent_id): + run_id = event.get("run_id") + if run_id and announced["run_id"] != run_id: + announced["run_id"] = run_id + print(f"[bold cyan][{_agent_id}][/bold cyan] RUN_ID=[bold]{run_id}[/bold]") + print(f" progress: python -m evaluator.cli progress {run_id} --events 100") + engine=EvaluationEngine(progress_callback=announce_run) + results.append(await engine.run_agent( + agent, start, end, source=source, limit=limit, reference_date=reference_date + )) + return results + print(asyncio.run(main())) + +@app.command('progress') +def progress(run_id: str, events: int=20): + print(asyncio.run(EvaluationRepository(auto_init_schema=False).aget_run_progress(run_id, event_limit=events))) + +@app.command('runs') +def runs(limit: int=20): + print(asyncio.run(EvaluationRepository(auto_init_schema=False).alist_runs(limit))) + +if __name__ == '__main__': + app() diff --git a/evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc b/evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc new file mode 100644 index 0000000..4b8bb9a Binary files /dev/null and b/evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc differ diff --git a/evaluator/collectors/__pycache__/base.cpython-313.pyc b/evaluator/collectors/__pycache__/base.cpython-313.pyc new file mode 100644 index 0000000..6727b81 Binary files /dev/null and b/evaluator/collectors/__pycache__/base.cpython-313.pyc differ diff --git a/evaluator/collectors/__pycache__/langfuse.cpython-313.pyc b/evaluator/collectors/__pycache__/langfuse.cpython-313.pyc new file mode 100644 index 0000000..ed6ee05 Binary files /dev/null and b/evaluator/collectors/__pycache__/langfuse.cpython-313.pyc differ diff --git a/evaluator/collectors/__pycache__/langfuse_credentials.cpython-313.pyc b/evaluator/collectors/__pycache__/langfuse_credentials.cpython-313.pyc new file mode 100644 index 0000000..a0d29f2 Binary files /dev/null and b/evaluator/collectors/__pycache__/langfuse_credentials.cpython-313.pyc differ diff --git a/evaluator/collectors/__pycache__/mock.cpython-313.pyc b/evaluator/collectors/__pycache__/mock.cpython-313.pyc new file mode 100644 index 0000000..40c1db8 Binary files /dev/null and b/evaluator/collectors/__pycache__/mock.cpython-313.pyc differ diff --git a/evaluator/collectors/agent_framework.py b/evaluator/collectors/agent_framework.py new file mode 100644 index 0000000..1b5602a --- /dev/null +++ b/evaluator/collectors/agent_framework.py @@ -0,0 +1,42 @@ +from __future__ import annotations +from datetime import datetime +from evaluator.collectors.base import ConversationCollector +from evaluator.core.models import ConversationRecord, ConversationMessage +from evaluator.persistence.oracle_store import OracleStore, _json_loads +from evaluator.config.settings import settings + +class AgentFrameworkCollector(ConversationCollector): + def __init__(self): + self.store = OracleStore(settings, auto_init_schema=False) + + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None): + return await self.store.to_thread(self._collect, period_start, period_end, agent_aliases or set(), limit or 100) + + def _collect(self, period_start, period_end, aliases, limit): + records=[] + with self.store.connect() as conn: + cur=conn.cursor() + cur.execute(f""" + select * from ( + select SESSION_ID, AGENT_ID, CHANNEL, CONTEXT_JSON, METADATA_JSON, CREATED_AT + from {self.store.t('AGENT_SESSION')} + where CREATED_AT >= :start_at and CREATED_AT < :end_at + order by CREATED_AT desc + ) where rownum <= :max_rows + """, dict(start_at=period_start, end_at=period_end, max_rows=limit)) + sessions=cur.fetchall() + for session_id, agent_id, channel, ctx, meta, created_at in sessions: + if aliases and agent_id not in aliases: continue + cur.execute(f""" + select ROLE, CONTENT, METADATA_JSON, CREATED_AT, MESSAGE_ID + from {self.store.t('AGENT_MESSAGE')} + where SESSION_ID=:session_id order by CREATED_AT + """, dict(session_id=session_id)) + rows=cur.fetchall() + msgs=[] + for role, content, msg_meta, msg_created, message_id in rows: + msgs.append(ConversationMessage(role=role, content=content or '', created_at=str(msg_created), metadata=_json_loads(msg_meta.read() if hasattr(msg_meta,'read') else msg_meta,{}))) + input_text=next((m.content for m in msgs if m.role in ('user','human')), '') + output_text=next((m.content for m in reversed(msgs) if m.role in ('assistant','ai','agent')), '') + records.append(ConversationRecord(session_id=session_id, trace_id=session_id, message_id=rows[-1][4] if rows else None, agent_id=agent_id, channel=channel, input_text=input_text, output_text=output_text, messages=msgs, metadata=_json_loads(meta.read() if hasattr(meta,'read') else meta,{}), raw={})) + return records diff --git a/evaluator/collectors/base.py b/evaluator/collectors/base.py new file mode 100644 index 0000000..cc00c1f --- /dev/null +++ b/evaluator/collectors/base.py @@ -0,0 +1,8 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from datetime import datetime +from evaluator.core.models import ConversationRecord + +class ConversationCollector(ABC): + @abstractmethod + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None) -> list[ConversationRecord]: ... diff --git a/evaluator/collectors/langfuse.py b/evaluator/collectors/langfuse.py new file mode 100644 index 0000000..65c28e6 --- /dev/null +++ b/evaluator/collectors/langfuse.py @@ -0,0 +1,518 @@ +from __future__ import annotations + +import json +import math +import random +from datetime import datetime +from typing import Any, Iterable + +import httpx + +from evaluator.collectors.base import ConversationCollector +from evaluator.collectors.langfuse_credentials import LangfuseCredentials, resolve_langfuse_credentials +from evaluator.config.settings import settings +from evaluator.core.models import ConversationMessage, ConversationRecord +from evaluator.identity.resolver import IdentityResolver +from evaluator.time_utils import to_utc_z + + +def _iso_z(dt: datetime) -> str: + return to_utc_z(dt) + + +def _metadata(obj: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(obj, dict): + return {} + meta = obj.get("metadata") or {} + return meta if isinstance(meta, dict) else {} + + +def _first_value(*values: Any) -> Any: + for value in values: + if value not in (None, "", [], {}): + return value + return None + + +def _content_to_text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, (int, float, bool)): + return str(value) + if isinstance(value, list): + return "\n".join(filter(None, (_content_to_text(v) for v in value))) + if isinstance(value, dict): + for key in ("text", "content", "message", "value", "input", "output", "completion"): + if key in value: + text = _content_to_text(value.get(key)) + if text: + return text + choices = value.get("choices") + if isinstance(choices, list) and choices: + return _content_to_text(choices[0]) + return "" + return str(value) + + +def _raw_character_count(value: Any) -> int: + """Character-count equivalent of len(trace.input) without losing structured input.""" + if value is None: + return 0 + if isinstance(value, str): + return len(value) + try: + return len(json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)) + except Exception: + return len(str(value)) + + +def _messages_from_value(value: Any, default_role: str) -> list[ConversationMessage]: + if value in (None, "", [], {}): + return [] + if isinstance(value, str): + text = value.strip() + return [ConversationMessage(role=default_role, content=text)] if text else [] + if isinstance(value, dict): + for key in ("messages", "conversation", "chat"): + if isinstance(value.get(key), list): + return _messages_from_value(value[key], default_role) + if "role" in value and "content" in value: + role = str(value.get("role") or default_role) + content = _content_to_text(value.get("content")).strip() + return [ConversationMessage(role=role, content=content, metadata={"source": "langfuse"})] if content else [] + text = _content_to_text(value).strip() + return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else [] + if isinstance(value, list): + out: list[ConversationMessage] = [] + for item in value: + out.extend(_messages_from_value(item, default_role)) + return out + text = _content_to_text(value).strip() + return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else [] + + +def _agent_id(trace: dict[str, Any]) -> str | None: + meta = _metadata(trace) + return meta.get("agent_id") or meta.get("agentId") or meta.get("agent") or trace.get("name") + + +def _trace_sort_key(trace: dict[str, Any]) -> str: + return str(trace.get("timestamp") or trace.get("createdAt") or trace.get("created_at") or trace.get("startTime") or "") + + +def _observation_sort_key(obs: dict[str, Any]) -> str: + return str(obs.get("startTime") or obs.get("start_time") or obs.get("createdAt") or obs.get("timestamp") or "") + + +def _session_entity_id(obj: dict[str, Any]) -> str | None: + """Return the identifier of a Langfuse Session entity. + + The generic ``id`` fallback is valid only for objects returned by + ``/api/public/sessions``. It must never be used for traces because a trace + id is not a conversation/session id. + """ + value = obj.get("sessionId") or obj.get("session_id") or obj.get("id") + return str(value) if value else None + + +def _session_id_from_metadata(obj: dict[str, Any] | None) -> str | None: + meta = _metadata(obj) + value = _first_value( + meta.get("session_id"), + meta.get("sessionId"), + meta.get("conversation_key"), + meta.get("conversationKey"), + ) + if value: + return str(value) + + business_context = meta.get("business_context") + if isinstance(business_context, dict): + value = _first_value( + business_context.get("session_id"), + business_context.get("sessionId"), + business_context.get("conversation_key"), + business_context.get("conversationKey"), + business_context.get("session_key"), + business_context.get("sessionKey"), + ) + if value: + return str(value) + return None + + +def _trace_session_id(trace: dict[str, Any], observations: list[dict[str, Any]] | None = None) -> str | None: + """Resolve the real conversation session id for a trace. + + Precedence: + 1. Native Langfuse trace.sessionId/session_id. + 2. Trace metadata session/conversation identifiers. + 3. Observation metadata session/conversation identifiers. + + Deliberately *never* falls back to trace.id. + """ + value = trace.get("sessionId") or trace.get("session_id") + if value: + return str(value) + + value = _session_id_from_metadata(trace) + if value: + return value + + for obs in observations or []: + value = _session_id_from_metadata(obs) + if value: + return value + return None + + +def _recovered_session(trace: dict[str, Any], observations: list[dict[str, Any]], session_id: str) -> dict[str, Any]: + """Build minimal session context for historical traces lacking native Langfuse sessions.""" + merged_meta: dict[str, Any] = {} + for obs in observations: + merged_meta.update(_metadata(obs)) + # Trace metadata has precedence over observation-derived fallback metadata. + merged_meta.update(_metadata(trace)) + merged_meta.setdefault("session_id", session_id) + return {"id": session_id, "sessionId": session_id, "metadata": merged_meta, "recovered": True} + + +def _items(payload: Any, keys: Iterable[str]) -> list[dict[str, Any]]: + if isinstance(payload, list): + return [x for x in payload if isinstance(x, dict)] + if not isinstance(payload, dict): + return [] + for key in keys: + value = payload.get(key) + if isinstance(value, list): + return [x for x in value if isinstance(x, dict)] + data = payload.get("data") + if isinstance(data, list): + return [x for x in data if isinstance(x, dict)] + return [] + + +class LangfuseCollector(ConversationCollector): + """TIM-compliant Langfuse v3 collector. + + Collection is session-oriented and bulk/paginated: + 1. Read every native Langfuse session in the BRT-derived UTC window. + 2. Read every trace and observation in the same window, once, page by page. + 3. Recover historical session ids from trace/observation metadata when the + native Langfuse session relation is absent. + 4. Deterministically sample the resulting logical session ids. + 5. Keep all traces belonging to sampled sessions. + + This preserves session-level semantics while eliminating the previous + trace-detail + observations N+1 pattern. + """ + + PAGE_SIZE = 100 + + def __init__(self, agent_id: str | None = None): + self.identity_resolver = IdentityResolver(settings.identity_config_path) + self.agent_id = agent_id + + async def _get_page(self, client: httpx.AsyncClient, path: str, params: dict[str, Any], creds: LangfuseCredentials) -> Any: + response = await client.get(path, params=params, auth=creds.auth) + if response.status_code >= 400: + raise RuntimeError(f"Langfuse API failed {response.status_code} {path}: {response.text}") + return response.json() + + async def _fetch_all_pages( + self, + client: httpx.AsyncClient, + path: str, + params: dict[str, Any], + keys: tuple[str, ...], + creds: LangfuseCredentials, + ) -> list[dict[str, Any]]: + page = 1 + out: list[dict[str, Any]] = [] + previous_signature: tuple[str, ...] | None = None + while True: + page_params = {**params, "page": page, "limit": self.PAGE_SIZE} + payload = await self._get_page(client, path, page_params, creds) + batch = _items(payload, keys) + if not batch: + break + + signature = tuple(str(x.get("id") or x.get("traceId") or x.get("sessionId") or "") for x in batch[:5]) + if page > 1 and signature == previous_signature: + # Protect against a legacy server ignoring the page parameter. + break + previous_signature = signature + out.extend(batch) + + meta = payload.get("meta") if isinstance(payload, dict) and isinstance(payload.get("meta"), dict) else {} + total_pages = meta.get("totalPages") or meta.get("total_pages") + if total_pages is not None and page >= int(total_pages): + break + total_items = meta.get("totalItems") or meta.get("total_items") + if total_items is not None and len(out) >= int(total_items): + break + if len(batch) < self.PAGE_SIZE: + break + page += 1 + return out + + async def _fetch_sessions(self, client: httpx.AsyncClient, start: datetime, end: datetime, creds: LangfuseCredentials) -> list[dict[str, Any]]: + params = {"fromTimestamp": _iso_z(start), "toTimestamp": _iso_z(end)} + return await self._fetch_all_pages(client, "/api/public/sessions", params, ("sessions",), creds) + + async def _fetch_traces(self, client: httpx.AsyncClient, start: datetime, end: datetime, creds: LangfuseCredentials) -> list[dict[str, Any]]: + params = {"fromTimestamp": _iso_z(start), "toTimestamp": _iso_z(end)} + return await self._fetch_all_pages(client, "/api/public/traces", params, ("traces",), creds) + + async def _fetch_observations(self, client: httpx.AsyncClient, start: datetime, end: datetime, creds: LangfuseCredentials) -> list[dict[str, Any]]: + # Legacy v3 deployments differ in accepted time-filter names. Try the two + # known public-API conventions, but never fall back to one request per trace. + candidates = [ + {"fromStartTime": _iso_z(start), "toStartTime": _iso_z(end)}, + {"fromTimestamp": _iso_z(start), "toTimestamp": _iso_z(end)}, + ] + last_error: Exception | None = None + for params in candidates: + try: + return await self._fetch_all_pages(client, "/api/public/observations", params, ("observations",), creds) + except Exception as exc: + last_error = exc + # Traces themselves normally carry input/output. Observations enrich context; + # a legacy deployment that cannot bulk-filter observations must not regress to N+1. + if last_error: + return [] + return [] + + @staticmethod + def _sample_session_ids(session_ids: list[str], percentage: float, seed: str) -> list[str]: + unique = sorted(set(x for x in session_ids if x)) + if not unique: + return [] + if percentage >= 1: + return unique + if percentage <= 0: + return [] + count = max(1, math.ceil(len(unique) * percentage)) + selected = random.Random(seed).sample(unique, min(count, len(unique))) + return sorted(selected) + + async def collect_sampled( + self, + period_start: datetime, + period_end: datetime, + *, + agent_aliases: set[str] | None, + percentage: float, + seed: str, + limit: int | None = None, + ) -> tuple[list[ConversationRecord], list[str], dict[str, Any]]: + if not settings.enable_langfuse: + raise RuntimeError("Langfuse disabled. Set ENABLE_LANGFUSE=true.") + creds = resolve_langfuse_credentials(self.agent_id) + aliases = {a for a in (agent_aliases or set()) if a} + + async with httpx.AsyncClient(base_url=creds.host, timeout=90) as client: + sessions = await self._fetch_sessions(client, period_start, period_end, creds) + session_map = {sid: s for s in sessions if (sid := _session_entity_id(s))} + native_session_ids = set(session_map) + + # Bulk-read traces and observations once for the full day. Observations + # are needed before sampling because historical/legacy traces may carry + # the real session id only in metadata instead of native trace.sessionId. + # This remains O(pages), never O(traces): no per-trace API calls. + traces = await self._fetch_traces(client, period_start, period_end, creds) + observations = await self._fetch_observations(client, period_start, period_end, creds) + obs_by_trace: dict[str, list[dict[str, Any]]] = {} + for obs in observations: + tid = obs.get("traceId") or obs.get("trace_id") + if tid: + obs_by_trace.setdefault(str(tid), []).append(obs) + + use_alias_filter = bool(aliases) + eligible_session_ids: set[str] = set() + trace_session_ids: dict[str, str] = {} + recovered_session_ids: set[str] = set() + traces_without_session = 0 + + for trace in traces: + if str(trace.get("name") or "").startswith("judge-job-"): + continue + aid = _agent_id(trace) + if use_alias_filter and (not aid or aid not in aliases): + continue + + tid = str(trace.get("id") or "") + trace_observations = obs_by_trace.get(tid, []) + sid = _trace_session_id(trace, trace_observations) + if not sid: + traces_without_session += 1 + continue + + trace_session_ids[tid] = sid + eligible_session_ids.add(sid) + if sid not in session_map: + session_map[sid] = _recovered_session(trace, trace_observations, sid) + recovered_session_ids.add(sid) + + sampled_ids = self._sample_session_ids(sorted(eligible_session_ids), percentage, seed) + sampled_set = set(sampled_ids) + selected_traces: list[dict[str, Any]] = [] + for trace in traces: + if str(trace.get("name") or "").startswith("judge-job-"): + continue + aid = _agent_id(trace) + if use_alias_filter and (not aid or aid not in aliases): + continue + tid = str(trace.get("id") or "") + sid = trace_session_ids.get(tid) + if not sid or sid not in sampled_set: + continue + selected_traces.append(trace) + + selected_traces.sort(key=_trace_sort_key) + if limit is not None: + selected_traces = selected_traces[: max(0, limit)] + + records: list[ConversationRecord] = [] + for trace in selected_traces: + tid = str(trace.get("id") or "") + sid = trace_session_ids.get(tid) or "" + record = self._to_record( + trace, + obs_by_trace.get(tid, []), + session_map.get(sid, {}), + _agent_id(trace), + resolved_session_id=sid, + ) + if record.input_text or record.output_text or record.messages: + records.append(record) + + stats = { + "credential_source": creds.source, + "sessions_read": len(session_map), + "native_sessions_read": len(native_session_ids), + "metadata_sessions_recovered": len(recovered_session_ids), + "traces_without_session": traces_without_session, + "sampled_sessions": len(sampled_ids), + "traces_read": len(traces), + "selected_traces": len(selected_traces), + "observations_read": len(observations), + } + return records, sampled_ids, stats + + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None) -> list[ConversationRecord]: + # Generic/manual command: keep all sessions, still using the same full-day + # paginated path and no N+1 behavior. + records, _, _ = await self.collect_sampled( + period_start, + period_end, + agent_aliases=agent_aliases, + percentage=1.0, + seed="manual", + limit=limit, + ) + return records + + def _to_record( + self, + trace: dict[str, Any], + observations: list[dict[str, Any]], + session: dict[str, Any], + agent_id: str | None, + resolved_session_id: str | None = None, + ) -> ConversationRecord: + trace_id = str(trace.get("id") or "") + session_id = ( + resolved_session_id + or _trace_session_id(trace, observations) + or _session_entity_id(session) + or "" + ) + session_meta = _metadata(session) + trace_meta = _metadata(trace) + observations = sorted(observations, key=_observation_sort_key) + + input_value = trace.get("input") + output_value = trace.get("output") + messages = _messages_from_value(input_value, "user") + _messages_from_value(output_value, "assistant") + for obs in observations: + messages.extend(_messages_from_value(obs.get("input"), "user")) + messages.extend(_messages_from_value(obs.get("output"), "assistant")) + + # Stable de-duplication while preserving chronological order. + deduped: list[ConversationMessage] = [] + seen: set[tuple[str, str]] = set() + for msg in messages: + key = (msg.role, msg.content) + if msg.content and key not in seen: + seen.add(key) + deduped.append(msg) + messages = deduped + + input_text = _content_to_text(input_value).strip() + output_text = _content_to_text(output_value).strip() + if not input_text: + input_text = next((m.content for m in messages if m.role.lower() in {"user", "human"}), "") + if not output_text: + output_text = next((m.content for m in reversed(messages) if m.role.lower() in {"assistant", "agent", "ai"}), "") + + trace_input = input_value if isinstance(input_value, dict) else {} + session_bc = session_meta.get("business_context") if isinstance(session_meta.get("business_context"), dict) else {} + trace_bc = trace_input.get("business_context") if isinstance(trace_input.get("business_context"), dict) else {} + identity_payload = { + **session_meta, + **trace_meta, + "business_context": {**session_bc, **trace_bc}, + "session_id": session_id, + "message_id": _first_value(trace_meta.get("message_id"), session_meta.get("message_id")), + "conversation_key": _first_value(trace_meta.get("conversation_key"), session_meta.get("conversation_key")), + } + business_context = self.identity_resolver.resolve(identity_payload) + + # Requirement: trace metadata first, session metadata as fallback. + ura_call_id = _first_value( + trace_meta.get("ura_call_id"), trace_meta.get("uraCallId"), + session_meta.get("ura_call_id"), session_meta.get("uraCallId"), + business_context.get("interaction_key"), + ) + channel_id = _first_value( + trace_meta.get("channel_id"), trace_meta.get("channelId"), trace_meta.get("channel"), + session_meta.get("channel_id"), session_meta.get("channelId"), session_meta.get("channel"), + ) + message_id = _first_value( + trace_meta.get("message_id"), trace_meta.get("messageId"), + session_meta.get("message_id"), session_meta.get("messageId"), + business_context.get("interaction_key"), trace_id, + ) + + metadata = { + **session_meta, + **trace_meta, + "business_context": business_context, + "ura_call_id": ura_call_id, + "channel_id": channel_id, + "session_metadata": session_meta, + } + raw = { + "trace": trace, + "session": session, + "observations": observations, + "trace_input_raw": input_value, + "prompt_length": _raw_character_count(input_value), + } + return ConversationRecord( + trace_id=trace_id, + # Preserve the resolved Langfuse/conversation session id. Business-context + # session_key may be a shortened/raw key and must not replace sessionId. + session_id=session_id, + message_id=str(message_id) if message_id is not None else None, + agent_id=agent_id, + channel=str(channel_id or "web"), + input_text=input_text, + output_text=output_text, + messages=messages, + metadata=metadata, + raw=raw, + ) diff --git a/evaluator/collectors/langfuse_credentials.py b/evaluator/collectors/langfuse_credentials.py new file mode 100644 index 0000000..963059e --- /dev/null +++ b/evaluator/collectors/langfuse_credentials.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from evaluator.config.settings import settings + + +@dataclass(frozen=True) +class LangfuseCredentials: + host: str + public_key: str + secret_key: str + source: str + + @property + def auth(self) -> tuple[str, str]: + return self.public_key, self.secret_key + + +def _read_secret_file(path: str | None) -> str | None: + if not path: + return None + p = Path(path).expanduser() + if not p.is_absolute(): + p = settings.path(p) + if not p.exists(): + return None + value = p.read_text(encoding="utf-8").strip() + return value or None + + +def _resolve_value(entry: dict[str, Any], direct_key: str, env_key: str, file_key: str) -> str | None: + """Resolve a credential field without imposing any agent-specific naming convention. + + Resolution order is intentionally declarative: + 1. direct value in the credentials definition file + 2. environment variable whose *name* is declared in that file + 3. file path whose *path* is declared in that file + + This lets Kubernetes mount the complete credentials YAML as a Secret, or lets + teams keep only references to environment variables/secret files in Git. + """ + direct = entry.get(direct_key) + if direct is not None and str(direct).strip(): + return str(direct).strip() + + env_name = entry.get(env_key) + if env_name: + value = os.getenv(str(env_name)) + if value and value.strip(): + return value.strip() + + file_name = entry.get(file_key) + return _read_secret_file(str(file_name)) if file_name else None + + +def load_langfuse_credential_definitions(path: str | None = None) -> dict[str, LangfuseCredentials]: + """Load all configured Langfuse projects and index them by logical agentId. + + No agent names are hardcoded here. The function loops over whatever entries + are present in the definition file. + """ + config_path = settings.path(path or settings.langfuse_credentials_config_path) + if not config_path.exists(): + raise RuntimeError( + f"Langfuse credentials definition file not found: {config_path}. " + "Set LANGFUSE_CREDENTIALS_CONFIG_PATH to a mounted YAML file." + ) + + data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + root = data.get("langfuse", data) + entries = root.get("agents", []) if isinstance(root, dict) else [] + + # Also accept mapping form for operational convenience: + # agents:\n my-agent: {host: ..., publicKey: ..., secretKey: ...} + if isinstance(entries, dict): + normalized = [] + for agent_id, item in entries.items(): + item = dict(item or {}) + item.setdefault("agentId", agent_id) + normalized.append(item) + entries = normalized + + resolved: dict[str, LangfuseCredentials] = {} + for item in entries or []: + if not isinstance(item, dict): + continue + agent_id = item.get("agentId", item.get("agent_id")) + if not agent_id: + raise ValueError("Each Langfuse credential entry must define agentId or agent_id") + agent_id = str(agent_id) + + host = _resolve_value(item, "host", "hostEnv", "hostFile") or settings.langfuse_host + public_key = _resolve_value(item, "publicKey", "publicKeyEnv", "publicKeyFile") + secret_key = _resolve_value(item, "secretKey", "secretKeyEnv", "secretKeyFile") + + if not public_key or not secret_key: + raise RuntimeError( + f"Incomplete Langfuse credentials for agent {agent_id!r} in {config_path}. " + "Configure publicKey/secretKey directly or their Env/File references." + ) + + resolved[agent_id] = LangfuseCredentials( + host=str(host).rstrip("/"), + public_key=public_key, + secret_key=secret_key, + source=f"{config_path}#{agent_id}", + ) + + return resolved + + +def resolve_langfuse_credentials( + agent_id: str | None = None, + *, + allow_global_fallback: bool | None = None, +) -> LangfuseCredentials: + """Resolve credentials from a declarative file, never from hardcoded agent names.""" + if allow_global_fallback is None: + allow_global_fallback = agent_id is None + + if agent_id: + definitions = load_langfuse_credential_definitions() + creds = definitions.get(agent_id) + if creds: + return creds + configured = ", ".join(sorted(definitions)) or "" + raise RuntimeError( + f"No Langfuse credentials configured for agent {agent_id!r}. " + f"Configured agents: {configured}." + ) + + if allow_global_fallback and settings.langfuse_public_key and settings.langfuse_secret_key: + return LangfuseCredentials( + settings.langfuse_host.rstrip("/"), + settings.langfuse_public_key, + settings.langfuse_secret_key, + "LANGFUSE_* (manual/global fallback)", + ) + + raise RuntimeError( + "Langfuse credentials missing. For run-agents configure the declarative credentials file; " + "for the generic manual run, LANGFUSE_PUBLIC_KEY/LANGFUSE_SECRET_KEY may be used." + ) diff --git a/evaluator/collectors/mock.py b/evaluator/collectors/mock.py new file mode 100644 index 0000000..3303001 --- /dev/null +++ b/evaluator/collectors/mock.py @@ -0,0 +1,9 @@ +from __future__ import annotations +from datetime import datetime +from evaluator.collectors.base import ConversationCollector +from evaluator.core.models import ConversationRecord, ConversationMessage + +class MockCollector(ConversationCollector): + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None=None, limit: int | None=None): + agent = next(iter(agent_aliases), 'mock-agent') if agent_aliases else 'mock-agent' + return [ConversationRecord(trace_id='mock-trace-1', session_id='mock-session-1', message_id='mock-message-1', agent_id=agent, channel='web', input_text='quero minha fatura', output_text='Sua fatura está em aberto no valor de R$ 120.', messages=[ConversationMessage(role='user', content='quero minha fatura'), ConversationMessage(role='assistant', content='Sua fatura está em aberto no valor de R$ 120.')], metadata={'mock': True})] diff --git a/evaluator/config/__pycache__/agents.cpython-313.pyc b/evaluator/config/__pycache__/agents.cpython-313.pyc new file mode 100644 index 0000000..78872dc Binary files /dev/null and b/evaluator/config/__pycache__/agents.cpython-313.pyc differ diff --git a/evaluator/config/__pycache__/settings.cpython-313.pyc b/evaluator/config/__pycache__/settings.cpython-313.pyc new file mode 100644 index 0000000..deb131d Binary files /dev/null and b/evaluator/config/__pycache__/settings.cpython-313.pyc differ diff --git a/evaluator/config/agents.py b/evaluator/config/agents.py new file mode 100644 index 0000000..1c15171 --- /dev/null +++ b/evaluator/config/agents.py @@ -0,0 +1,39 @@ +from __future__ import annotations +from dataclasses import dataclass, field +import yaml +from pathlib import Path +from evaluator.config.settings import settings + +@dataclass +class AgentConfig: + agent_id: str + enabled: bool = True + days_back: int = 1 + percentage: float = 1.0 + langfuse_agent_aliases: list[str] = field(default_factory=list) + gcs_prefix: str = "" + + @property + def aliases(self) -> set[str]: + return {self.agent_id, *self.langfuse_agent_aliases} + + +def load_agents(path: str | None = None) -> list[AgentConfig]: + p = settings.path(path or settings.agents_config_path) + data = yaml.safe_load(p.read_text()) or {} + agents = [] + for item in data.get("agents", []): + agent_id = item.get("agent_id", item.get("agentId")) + if not agent_id: + raise ValueError("Each agent entry must define agentId or agent_id") + cfg = AgentConfig( + agent_id=str(agent_id), + enabled=bool(item.get("enabled", True)), + days_back=int(item.get("days_back", item.get("daysBack", 1))), + percentage=float(item.get("percentage", 1.0)), + langfuse_agent_aliases=list(item.get("langfuse_agent_aliases", [])), + gcs_prefix=str(item.get("gcs_prefix", "")), + ) + if cfg.enabled: + agents.append(cfg) + return agents diff --git a/evaluator/config/settings.py b/evaluator/config/settings.py new file mode 100644 index 0000000..060e8d2 --- /dev/null +++ b/evaluator/config/settings.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import os +from pathlib import Path +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict +from dotenv import load_dotenv + +ROOT_DIR = Path(__file__).resolve().parents[2] +ENV_PATH = ROOT_DIR / ".env" +load_dotenv(ENV_PATH, override=True) + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=str(ENV_PATH), extra="ignore", case_sensitive=False) + + app_env: str = Field(default="dev", validation_alias="APP_ENV") + + adb_user: str = Field(default="", validation_alias="ADB_USER") + adb_password: str = Field(default="", validation_alias="ADB_PASSWORD") + adb_dsn: str = Field(default="", validation_alias="ADB_DSN") + adb_wallet_location: str | None = Field(default=None, validation_alias="ADB_WALLET_LOCATION") + adb_wallet_password: str | None = Field(default=None, validation_alias="ADB_WALLET_PASSWORD") + adb_table_prefix: str = Field(default="AGENTFW", validation_alias="ADB_TABLE_PREFIX") + + enable_langfuse: bool = Field(default=False, validation_alias="ENABLE_LANGFUSE") + langfuse_public_key: str | None = Field(default=None, validation_alias="LANGFUSE_PUBLIC_KEY") + langfuse_secret_key: str | None = Field(default=None, validation_alias="LANGFUSE_SECRET_KEY") + langfuse_host: str = Field(default="http://localhost:3005", validation_alias="LANGFUSE_HOST") + publish_langfuse_scores: bool = Field(default=False, validation_alias="PUBLISH_LANGFUSE_SCORES") + langfuse_credentials_config_path: str = Field( + default="configs/langfuse/langfuse_credentials.yaml", + validation_alias="LANGFUSE_CREDENTIALS_CONFIG_PATH", + ) + + # llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER") + # llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE") + # llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH") + # oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT") + # oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID") + # oci_genai_compartment_id: str | None = Field(default=None, validation_alias="OCI_GENAI_COMPARTMENT_ID") + # oci_genai_auth_type: str = Field(default="api_key", validation_alias="OCI_GENAI_AUTH_TYPE") + # oci_config_path: str | None = Field(default=None, validation_alias="OCI_CONFIG_PATH") + # oci_config_profile: str = Field(default="DEFAULT", validation_alias="OCI_CONFIG_PROFILE") + # llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE") + # llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS") + + # LLM / OCI GenAI OpenAI-compatible, mesmo padrão do Agent Framework + llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER") + llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE") + llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH") + + oci_genai_base_url: str | None = Field(default=None, validation_alias="OCI_GENAI_BASE_URL") + oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT") # compatibilidade + oci_genai_model: str = Field(default="openai.gpt-4.1", validation_alias="OCI_GENAI_MODEL") + oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID") # compatibilidade + oci_genai_api_key: str | None = Field(default=None, validation_alias="OCI_GENAI_API_KEY") + oci_genai_project_ocid: str | None = Field(default=None, validation_alias="OCI_GENAI_PROJECT_OCID") + oci_genai_endpoint_id: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT_ID") + oci_auth_mode: str = Field(default="config_file", validation_alias="OCI_AUTH_MODE") + oci_config_file: str = Field(default="~/.oci/config", validation_alias="OCI_CONFIG_FILE") + oci_profile: str = Field(default="DEFAULT", validation_alias="OCI_PROFILE") + oci_compartment_id: str | None = Field(default=None, validation_alias="OCI_COMPARTMENT_ID") + oci_region: str = Field(default="sa-saopaulo-1", validation_alias="OCI_REGION") + + llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE") + llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS") + llm_timeout_seconds: int = Field(default=120, validation_alias="LLM_TIMEOUT_SECONDS") + + agents_config_path: str = Field(default="configs/judge/agents.yaml", validation_alias="AGENTS_CONFIG_PATH") + trace_prompt_path: str = Field(default="configs/judge/trace_metrics.yaml", validation_alias="TRACE_PROMPT_PATH") + session_prompt_path: str = Field(default="configs/judge/session_metrics.yaml", validation_alias="SESSION_PROMPT_PATH") + output_dir: str = Field(default="output", validation_alias="OUTPUT_DIR") + batch_size: int = Field(default=50, validation_alias="BATCH_SIZE") + max_attempts: int = Field(default=3, validation_alias="MAX_ATTEMPTS") + context_turn_limit: int = Field(default=6, validation_alias="CONTEXT_TURN_LIMIT") + enable_gcs_upload: bool = Field(default=False, validation_alias="ENABLE_GCS_UPLOAD") + judge_gcs_bucket: str | None = Field(default=None, validation_alias="JUDGE_GCS_BUCKET") + storage_config_path: str = Field(default="configs/judge/storage.yaml", validation_alias="STORAGE_CONFIG_PATH") + google_application_credentials: str | None = Field(default=None, validation_alias="GOOGLE_APPLICATION_CREDENTIALS") + identity_config_path: str = "configs/identity.yaml" + + @property + def project_root(self) -> Path: + return ROOT_DIR + + def path(self, value: str | Path) -> Path: + p = Path(value) + return p if p.is_absolute() else ROOT_DIR / p + + @property + def ADB_USER(self): return self.adb_user + @property + def ADB_PASSWORD(self): return self.adb_password + @property + def ADB_DSN(self): return self.adb_dsn + @property + def ADB_WALLET_LOCATION(self): return self.adb_wallet_location + @property + def ADB_WALLET_PASSWORD(self): return self.adb_wallet_password + @property + def ADB_TABLE_PREFIX(self): return (self.adb_table_prefix or "AGENTFW").upper().rstrip("_") + + @property + def has_langfuse_credentials(self) -> bool: + return bool(self.langfuse_public_key and self.langfuse_secret_key) + + @property + def can_use_langfuse(self) -> bool: + return bool(self.enable_langfuse and self.has_langfuse_credentials) + + @property + def can_publish_langfuse_scores(self) -> bool: + return bool(self.publish_langfuse_scores and self.can_use_langfuse) + + @property + def OCI_GENAI_BASE_URL(self) -> str | None: + return self.oci_genai_base_url or self.oci_genai_endpoint + + @property + def OCI_GENAI_MODEL(self) -> str: + return self.oci_genai_model_id or self.oci_genai_model + + @property + def OCI_GENAI_API_KEY(self) -> str | None: + return self.oci_genai_api_key + + @property + def OCI_GENAI_PROJECT_OCID(self) -> str | None: + return self.oci_genai_project_ocid + + @property + def LLM_PROVIDER(self) -> str: + return self.llm_provider + + @property + def LLM_TEMPERATURE(self) -> float: + return self.llm_temperature + + @property + def LLM_MAX_TOKENS(self) -> int: + return self.llm_max_tokens + + @property + def LLM_TIMEOUT_SECONDS(self) -> int: + return self.llm_timeout_seconds + + @property + def LLM_PROFILES_PATH(self) -> str: + return self.llm_profiles_path + +settings = Settings() +if settings.ADB_WALLET_LOCATION: + os.environ["TNS_ADMIN"] = settings.ADB_WALLET_LOCATION diff --git a/evaluator/core/__pycache__/models.cpython-313.pyc b/evaluator/core/__pycache__/models.cpython-313.pyc new file mode 100644 index 0000000..605f654 Binary files /dev/null and b/evaluator/core/__pycache__/models.cpython-313.pyc differ diff --git a/evaluator/core/models.py b/evaluator/core/models.py new file mode 100644 index 0000000..3191415 --- /dev/null +++ b/evaluator/core/models.py @@ -0,0 +1,56 @@ +from __future__ import annotations +from datetime import datetime +from enum import Enum +from pydantic import BaseModel, Field +from typing import Any + +class RunStatus(str, Enum): + RUNNING = "RUNNING" + COMPLETED = "COMPLETED" + PARTIAL = "PARTIAL" + FAILED = "FAILED" + +class ItemStatus(str, Enum): + PENDING = "PENDING" + PROCESSING = "PROCESSING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + SKIPPED = "SKIPPED" + +class ConversationMessage(BaseModel): + role: str + content: str + created_at: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + +class ConversationRecord(BaseModel): + trace_id: str | None = None + session_id: str + message_id: str | None = None + agent_id: str | None = None + channel: str | None = None + input_text: str = "" + output_text: str = "" + messages: list[ConversationMessage] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + raw: dict[str, Any] = Field(default_factory=dict) + +class TraceJudgeResult(BaseModel): + judgeScore: float | None = Field(default=None, ge=0.0, le=1.0) + accuracyScore: float | None = Field(default=None, ge=0.0, le=1.0) + alucinationScore: float | None = Field(default=None, ge=0.0, le=1.0) + rationale: str = "" + judge_name: str = "trace_metrics" + judge_type: str = "trace" + +class SessionJudgeResult(BaseModel): + inferredCsiScore: float = Field(ge=0.0, le=1.0) + resolution: int = Field(ge=0, le=1) + conversationPrecision: int = Field(ge=0, le=1) + rationale: str = "" + judge_name: str = "session_metrics" + judge_type: str = "session" + +class CombinedJudgeResult(BaseModel): + trace: TraceJudgeResult + session: SessionJudgeResult | None = None diff --git a/evaluator/engine.py b/evaluator/engine.py new file mode 100644 index 0000000..46e2c15 --- /dev/null +++ b/evaluator/engine.py @@ -0,0 +1,192 @@ +from __future__ import annotations +import asyncio, inspect, json, math, random +from datetime import date, datetime +from typing import Any, Awaitable, Callable +from evaluator.collectors.base import ConversationCollector +from evaluator.collectors.langfuse import LangfuseCollector +from evaluator.collectors.agent_framework import AgentFrameworkCollector +from evaluator.collectors.mock import MockCollector +from evaluator.config.agents import AgentConfig +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, RunStatus +from evaluator.judges.llm_judge import TIMStyleLLMJudge +from evaluator.output.legacy_exporter import export_legacy_txt_gz +from evaluator.persistence.repository import EvaluationRepository +from evaluator.publishers.langfuse_scores import LangfuseScorePublisher +from evaluator.publishers.langfuse_job_audit import LangfuseJobAuditPublisher +from evaluator.publishers.gcs_export import upload_evaluation_file + +ProgressCallback = Callable[[dict[str, Any]], Awaitable[None] | None] + + +class EvaluationEngine: + def __init__(self, repository: EvaluationRepository | None = None, progress_callback: ProgressCallback | None = None): + self.repository = repository or EvaluationRepository(auto_init_schema=False) + self.progress_callback = progress_callback + self.judge = TIMStyleLLMJudge() + self.langfuse_publisher = LangfuseScorePublisher() + self.job_audit = LangfuseJobAuditPublisher() + + async def _emit(self, progress_run_id: str, stage: str, message: str = "", **details): + details.pop("run_id", None) + await self.repository.arecord_progress(progress_run_id, stage, message, details) + event = {"run_id": progress_run_id, "stage": stage, "message": message, "details": details} + if self.progress_callback: + r = self.progress_callback(event) + if inspect.isawaitable(r): + await r + + def collector_for(self, source: str, agent_id: str | None = None) -> ConversationCollector: + if source == 'langfuse': return LangfuseCollector(agent_id=agent_id) + if source == 'agent_framework': return AgentFrameworkCollector() + if source == 'mock': return MockCollector() + raise ValueError('source must be langfuse, agent_framework or mock') + + async def _audit(self, progress_run_id: str, audit_trace_id: str, agent_id: str, status: str, period_start: datetime, period_end: datetime, *, required: bool = True, **details) -> bool: + try: + result = await self.job_audit.publish( + audit_trace_id, agent_id, status, period_start=period_start, period_end=period_end, details=details + ) + if not result or not result.get("ok"): + raise RuntimeError(f"Langfuse audit did not confirm persistence: {result!r}") + await self._emit(progress_run_id, 'LANGFUSE_JOB_AUDIT', f'judge-job-{agent_id} status={status}', audit_status=status, audit_result=result) + return True + except Exception as exc: + await self._emit(progress_run_id, 'LANGFUSE_JOB_AUDIT_FAILED', f'Langfuse job audit failed: {exc}', audit_status=status, error=str(exc)) + if required: + raise + return False + + async def run_agent(self, agent: AgentConfig, period_start: datetime, period_end: datetime, source: str = 'langfuse', limit: int | None = None, reference_date: date | None = None) -> dict: + run_id = await self.repository.acreate_run(period_start, period_end, source, agent.agent_id) + reference_date = reference_date or period_start.date() + audit_trace_id = self.job_audit.new_trace_id() + try: + # Emit RUN_CREATED immediately after persistence so CLI/monitor can expose + # the RUN_ID before any external Langfuse audit/network call is performed. + await self._emit(run_id, 'RUN_CREATED', f'Agent run created: {agent.agent_id}', agent_id=agent.agent_id, source=source, reference_date=reference_date.isoformat()) + await self._audit(run_id, audit_trace_id, agent.agent_id, 'created', period_start, period_end, run_id=run_id, source=source) + collector = self.collector_for(source, agent.agent_id) + await self._emit(run_id, 'COLLECTING', 'Collecting sessions and conversations') + seed = f'{agent.agent_id}:{reference_date.isoformat()}' + if isinstance(collector, LangfuseCollector): + records, sampled_sessions, collection_stats = await collector.collect_sampled( + period_start, period_end, agent_aliases=agent.aliases, percentage=agent.percentage, seed=seed, limit=limit + ) + await self._emit( + run_id, 'COLLECTED', f'Collected {len(records)} traces from {len(sampled_sessions)} sampled sessions', + **collection_stats + ) + else: + records = await collector.collect(period_start, period_end, agent_aliases=agent.aliases, limit=limit) + records, sampled_sessions = self._sample_sessions(records, agent.percentage, agent.agent_id, reference_date) + collection_stats = {} + await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} traces before session sampling') + await self._emit( + run_id, + 'SAMPLED', + f'Kept {len(sampled_sessions)} sessions / {len(records)} traces', + percentage=agent.percentage, + seed=seed, + sampled_sessions=sampled_sessions, + ) + inserted = await self.repository.ainsert_items(run_id, records) + await self._emit(run_id, 'ITEMS_INSERTED', f'Inserted {inserted} items') + summary = await self._process(run_id) + output_path = export_legacy_txt_gz(self.repository, run_id, agent.agent_id) + await self._emit(run_id, 'EXPORTED', f'Exported {output_path}', output_file=str(output_path)) + if not settings.enable_gcs_upload: + raise RuntimeError("TIM specification requires GCS upload for run-agents; set ENABLE_GCS_UPLOAD=true") + uploaded_to = await asyncio.to_thread(upload_evaluation_file, output_path, agent.gcs_prefix) + await self._emit(run_id, 'GCS_UPLOADED', f'Uploaded {uploaded_to}', uploaded_to=uploaded_to) + output_path.unlink() + await self._emit(run_id, 'LOCAL_FILE_DELETED', 'Local export deleted after successful GCS upload') + result = {**summary, 'agent_id': agent.agent_id, 'output_file': str(output_path), 'uploaded_to': uploaded_to} + # _audit() receives status as an explicit positional argument. The run + # summary also contains a "status" key (e.g. COMPLETED), so forwarding + # **result unchanged would pass status twice and raise: + # TypeError: _audit() got multiple values for argument 'status' + audit_details = dict(result) + # Never forward names that are explicit _audit() parameters. + for reserved_key in ('progress_run_id', 'audit_trace_id', 'agent_id', 'status', 'period_start', 'period_end'): + audit_details.pop(reserved_key, None) + await self._audit(run_id, audit_trace_id, agent.agent_id, 'completed', period_start, period_end, **audit_details) + return result + except Exception as exc: + await self.repository.amark_run_status(run_id, RunStatus.FAILED, str(exc)) + await self._emit(run_id, 'FAILED', f'Agent run failed: {exc}', error=str(exc)) + await self._audit(run_id, audit_trace_id, agent.agent_id, 'failed', period_start, period_end, required=False, run_id=run_id, error=str(exc)) + return {'status': 'FAILED', 'run_id': run_id, 'agent_id': agent.agent_id, 'error': str(exc)} + + async def run(self, period_start: datetime, period_end: datetime, source: str = 'langfuse', limit: int | None = None) -> dict: + run_id = await self.repository.acreate_run(period_start, period_end, source, None) + try: + collector = self.collector_for(source) + await self._emit(run_id, 'COLLECTING', 'Collecting conversations') + records = await collector.collect(period_start, period_end, limit=limit) + await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} records') + await self.repository.ainsert_items(run_id, records) + return await self._process(run_id) + except Exception as exc: + await self.repository.amark_run_status(run_id, RunStatus.PARTIAL, str(exc)) + await self._emit(run_id, 'PARTIAL', f'Run failed: {exc}', error=str(exc)) + return {'status': 'PARTIAL', 'run_id': run_id, 'error': str(exc)} + + async def _process(self, run_id: str) -> dict: + processed_records: list[ConversationRecord] = [] + while True: + items = await self.repository.afetch_next_items(run_id, settings.batch_size) + if not items: break + await self._emit(run_id, 'BATCH_STARTED', f'Processing {len(items)} items') + for item in items: + item_id = item['item_id'] + await self.repository.amark_item_processing(item_id) + try: + raw = item['raw_json'] + if hasattr(raw, 'read'): raw = raw.read() + record = ConversationRecord.model_validate(json.loads(raw)) + result = await self.judge.judge_trace(record) + await self.repository.asave_trace_result(run_id, item_id, record, result) + await self.langfuse_publisher.publish_trace_score(record, result) + await self.repository.amark_item_completed(run_id, item_id) + processed_records.append(record) + await self._emit( + run_id, + "ITEM_COMPLETED", + f"Item completed {item_id}", + trace_id=record.trace_id, + session_id=record.session_id, + judgeScore=result.judgeScore, + accuracyScore=result.accuracyScore, + alucinationScore=result.alucinationScore, + rationale=result.rationale, + ) + except Exception as exc: + await self.repository.amark_item_failed(run_id, item_id, str(exc)) + await self._emit(run_id, 'ITEM_FAILED', f'Item failed {item_id}', error=str(exc)) + if processed_records: + sessions = await self.judge.judge_sessions(processed_records) + for sid, result in sessions.items(): + agent_id = next((r.agent_id for r in processed_records if r.session_id == sid), None) + await self.repository.asave_session_result(run_id, sid, agent_id, result) + await self._emit(run_id, 'SESSION_JUDGE_COMPLETED', f'Evaluated {len(sessions)} sessions') + await self.repository.amark_run_status(run_id, RunStatus.COMPLETED) + summary = await self.repository.asummarize_run(run_id) + await self._emit(run_id, 'COMPLETED', 'Run completed', **summary) + return {'status': 'COMPLETED', **summary} + + def _sample_sessions(self, records: list[ConversationRecord], percentage: float, agent_id: str, reference_date: date) -> tuple[list[ConversationRecord], list[str]]: + """Deterministic session sampling; all traces from selected sessions are retained.""" + if not records: + return [], [] + session_ids = sorted({r.session_id for r in records}) + if percentage >= 1: + selected = session_ids + elif percentage <= 0: + selected = [] + else: + count = min(len(session_ids), max(1, math.ceil(len(session_ids) * percentage))) + rng = random.Random(f"{agent_id}:{reference_date.isoformat()}") + selected = sorted(rng.sample(session_ids, count)) + selected_set = set(selected) + return [r for r in records if r.session_id in selected_set], selected diff --git a/evaluator/identity/__pycache__/resolver.cpython-313.pyc b/evaluator/identity/__pycache__/resolver.cpython-313.pyc new file mode 100644 index 0000000..7c642c1 Binary files /dev/null and b/evaluator/identity/__pycache__/resolver.cpython-313.pyc differ diff --git a/evaluator/identity/resolver.py b/evaluator/identity/resolver.py new file mode 100644 index 0000000..a0ea2e2 --- /dev/null +++ b/evaluator/identity/resolver.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import yaml + + +def _deep_get(data: dict, path: str): + cur = data + for part in path.split("."): + if not isinstance(cur, dict): + return None + cur = cur.get(part) + return cur + + +class IdentityResolver: + def __init__(self, path: str = "configs/identity.yaml"): + self.path = Path(path) + self.config = yaml.safe_load(self.path.read_text(encoding="utf-8")) or {} + self.identity = self.config.get("identity", {}) + self.keys = self.identity.get("keys", {}) + + def resolve(self, payload: dict[str, Any]) -> dict[str, Any]: + out = {} + + for key, spec in self.keys.items(): + value = None + for source in spec.get("sources", []): + value = _deep_get(payload, source) + if value not in (None, ""): + break + out[key] = str(value) if value not in (None, "") else None + + out["metadata"] = { + "identity_version": self.identity.get("version"), + "identity_source": str(self.path), + } + + return out \ No newline at end of file diff --git a/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc b/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc new file mode 100644 index 0000000..214f163 Binary files /dev/null and b/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc differ diff --git a/evaluator/judges/llm_judge.py b/evaluator/judges/llm_judge.py new file mode 100644 index 0000000..2cb37f5 --- /dev/null +++ b/evaluator/judges/llm_judge.py @@ -0,0 +1,126 @@ +from __future__ import annotations +import json +import re +from collections import defaultdict +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, TraceJudgeResult, SessionJudgeResult +from evaluator.llm.client import LLMClient, create_llm_client +from evaluator.prompts.loader import load_prompt + + +def _json_from_text(text: str) -> dict: + try: + return json.loads(text) + except Exception: + m = re.search(r"\{.*\}", text, flags=re.S) + if not m: + raise + return json.loads(m.group(0)) + + +def _history(record: ConversationRecord, turn_limit: int | None = None, max_chars: int = 6000) -> str: + messages = record.messages or [] + if messages: + if turn_limit and turn_limit > 0: + # A turn is represented by one conversational message in the normalized record. + messages = messages[-turn_limit:] + text = "\n".join(f"{m.role}: {m.content}" for m in messages) + else: + text = f"user: {record.input_text}\nagent: {record.output_text}" + return text[-max_chars:] + + +class TIMStyleLLMJudge: + def __init__(self, llm: LLMClient | None = None): + self.llm = llm or create_llm_client() + self.trace_prompt = load_prompt(settings.trace_prompt_path, 'trace_metrics') + self.session_prompt = load_prompt(settings.session_prompt_path, 'session_metrics') + + @staticmethod + def _validated_trace_data(raw: str) -> dict: + data = _json_from_text(raw) + normalized = dict(data) + normalized.setdefault("judgeScore", normalized.get("judge_score")) + normalized.setdefault("accuracyScore", normalized.get("accuracy_score")) + normalized.setdefault("alucinationScore", normalized.get("alucination_score")) + normalized.setdefault("rationale", normalized.get("reasoning", "")) + # Pydantic validates the 0..1 contract here. Invalid metric output is treated + # like an invalid judge payload and receives the single specification retry. + TraceJudgeResult(**normalized) + return normalized + + async def _trace_json_with_retry(self, prompt: str) -> tuple[dict | None, str | None]: + """One normal attempt + one retry-hint for invalid JSON/metric contract.""" + raw = await self.llm.complete(prompt) + try: + return self._validated_trace_data(raw), None + except Exception as first_exc: + retry_prompt = f"""{prompt} + +RETRY-HINT: +A resposta anterior não pôde ser interpretada como JSON válido ou violou o contrato das métricas. +Retorne SOMENTE um objeto JSON válido, sem markdown, comentários ou texto adicional, +com judgeScore, accuracyScore e alucinationScore entre 0.0 e 1.0 e a chave rationale. +""" + retry_raw = await self.llm.complete(retry_prompt) + try: + return self._validated_trace_data(retry_raw), None + except Exception as second_exc: + return None, f"parse_failed_after_retry: {second_exc or first_exc}" + + async def judge_trace(self, record: ConversationRecord) -> TraceJudgeResult: + prompt = f"""{self.trace_prompt} + +HISTÓRICO: +{_history(record, turn_limit=settings.context_turn_limit)} + +MENSAGEM DO USUÁRIO: +{record.input_text} + +RESPOSTA DO AGENTE: +{record.output_text} + +METADATA: +{json.dumps(record.metadata, ensure_ascii=False, default=str)} +""" + data, parse_error = await self._trace_json_with_retry(prompt) + if data is None: + # Specification: after the second parse failure metrics become None; + # legacy exporter serializes those values as "0.00". + return TraceJudgeResult( + judgeScore=None, + accuracyScore=None, + alucinationScore=None, + rationale=parse_error or "parse_failed_after_retry", + ) + + data.setdefault("judge_name", "trace_metrics") + data.setdefault("judge_type", "trace") + data.setdefault("judgeScore", data.get("judge_score")) + data.setdefault("accuracyScore", data.get("accuracy_score")) + data.setdefault("alucinationScore", data.get("alucination_score")) + data.setdefault("rationale", data.get("reasoning", "")) + return TraceJudgeResult(**data) + + async def judge_sessions(self, records: list[ConversationRecord]) -> dict[str, SessionJudgeResult]: + grouped: dict[str, list[ConversationRecord]] = defaultdict(list) + for r in records: + grouped[r.session_id].append(r) + out = {} + for session_id, items in grouped.items(): + transcript = "\n".join(_history(r, max_chars=3000) for r in items)[-9000:] + prompt = f"""{self.session_prompt} + +TRANSCRIÇÃO DA SESSÃO: +{transcript} +""" + raw = await self.llm.complete(prompt) + data = _json_from_text(raw) + data.setdefault("judge_name", "session_metrics") + data.setdefault("judge_type", "session") + data.setdefault("inferredCsiScore", data.get("inferred_csi_score", 0)) + data.setdefault("resolution", data.get("resolution", 0)) + data.setdefault("conversationPrecision", data.get("conversation_precision", 0)) + data.setdefault("rationale", data.get("reasoning", "")) + out[session_id] = SessionJudgeResult(**data) + return out diff --git a/evaluator/llm/__pycache__/client.cpython-313.pyc b/evaluator/llm/__pycache__/client.cpython-313.pyc new file mode 100644 index 0000000..c40fe8e Binary files /dev/null and b/evaluator/llm/__pycache__/client.cpython-313.pyc differ diff --git a/evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc b/evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc new file mode 100644 index 0000000..aabdea1 Binary files /dev/null and b/evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc differ diff --git a/evaluator/llm/client.py b/evaluator/llm/client.py new file mode 100644 index 0000000..49fde57 --- /dev/null +++ b/evaluator/llm/client.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any + +from evaluator.config.settings import settings +from evaluator.llm.profile_resolver import LLMProfileResolver + + +class LLMClient: + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + raise NotImplementedError + + +class MockLLMClient(LLMClient): + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + if "inferredCsiScore" in prompt: + return json.dumps({ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "Avaliação mock." + }, ensure_ascii=False) + return json.dumps({ + "judgeScore": 0.7, + "accuracyScore": 0.7, + "alucinationScore": 0.1, + "rationale": "Avaliação mock." + }, ensure_ascii=False) + + +class OCICompatibleLLMClient(LLMClient): + def __init__(self): + self.resolver = LLMProfileResolver(settings) + + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + effective = self.resolver.resolve(profile_name or settings.llm_profile) + provider = str(effective.get("provider") or settings.LLM_PROVIDER).lower() + model = str(effective.get("model") or settings.OCI_GENAI_MODEL) + base_url = effective.get("base_url") or settings.OCI_GENAI_BASE_URL + api_key = effective.get("api_key") or settings.OCI_GENAI_API_KEY + temperature = effective.get("temperature", settings.LLM_TEMPERATURE) + max_tokens = effective.get("max_tokens", settings.LLM_MAX_TOKENS) + timeout = effective.get("timeout_seconds", settings.LLM_TIMEOUT_SECONDS) + + if provider == "mock": + return await MockLLMClient().complete(prompt, profile_name=profile_name) + if provider not in ("oci_openai", "openai_compatible"): + raise ValueError(f"Unsupported LLM provider: {provider}") + if not base_url: + raise RuntimeError("OCI_GENAI_BASE_URL is required for oci_openai provider") + if not api_key: + raise RuntimeError("OCI_GENAI_API_KEY is required for oci_openai provider") + + from openai import AsyncOpenAI + client = AsyncOpenAI(base_url=base_url, api_key=api_key, timeout=timeout) + resp = await client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=temperature, + max_tokens=max_tokens, + ) + return resp.choices[0].message.content or "" + + +class OCISDKLLMClient(LLMClient): + """OCI Generative AI native SDK, including dedicated endpoints.""" + + def __init__(self): + self.resolver = LLMProfileResolver(settings) + self._clients: dict[str, Any] = {} + + @staticmethod + def _normalize_endpoint(value: str | None) -> str: + endpoint = (value or "").strip().strip("'\"").rstrip("/") + if not endpoint: + region = settings.oci_region or "sa-saopaulo-1" + return f"https://inference.generativeai.{region}.oci.oraclecloud.com" + if "/openai/v1" in endpoint: + raise RuntimeError( + "OCI_GENAI_BASE_URL cannot contain /openai/v1 when LLM_PROVIDER=oci_sdk" + ) + return endpoint + + @staticmethod + def _config_and_signer(): + import oci + + mode = (settings.oci_auth_mode or "config_file").lower() + if mode == "config_file": + config_file = str(Path(settings.oci_config_file).expanduser()) + return oci.config.from_file(config_file, settings.oci_profile), None + if mode == "instance_principal": + return {}, oci.auth.signers.InstancePrincipalsSecurityTokenSigner() + if mode == "resource_principal": + return {}, oci.auth.signers.get_resource_principals_signer() + if mode in ("oke_workload_identity", "workload_identity"): + return {}, oci.auth.signers.get_oke_workload_identity_resource_principal_signer() + raise ValueError(f"Unsupported OCI_AUTH_MODE={settings.oci_auth_mode}") + + def _client(self, endpoint: str): + if endpoint not in self._clients: + from oci.generative_ai_inference import GenerativeAiInferenceClient + config, signer = self._config_and_signer() + kwargs: dict[str, Any] = {"config": config, "service_endpoint": endpoint} + if signer is not None: + kwargs["signer"] = signer + self._clients[endpoint] = GenerativeAiInferenceClient(**kwargs) + return self._clients[endpoint] + + @staticmethod + def _extract_answer(response: Any) -> str: + data = getattr(response, "data", response) + chat_response = getattr(data, "chat_response", None) or data + for attr in ("text", "message", "output_text"): + value = getattr(chat_response, attr, None) + if value: + return str(value) + choices = getattr(chat_response, "choices", None) or [] + if choices: + first = choices[0] + msg = getattr(first, "message", None) + content = getattr(msg, "content", None) if msg is not None else getattr(first, "text", None) + if isinstance(content, list): + return "".join(str(getattr(item, "text", item)) for item in content) + if content: + return str(content) + return str(chat_response) + + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + from oci.generative_ai_inference import models + + effective = self.resolver.resolve(profile_name or settings.llm_profile) + model = str(effective.get("model") or settings.OCI_GENAI_MODEL).strip() + endpoint_id = str(effective.get("endpoint_id") or settings.oci_genai_endpoint_id or "").strip() or None + if not endpoint_id and model.startswith("ocid1.generativeaiendpoint."): + endpoint_id = model + compartment_id = ( + effective.get("compartment_id") + or settings.oci_compartment_id + or settings.OCI_GENAI_PROJECT_OCID + ) + if not compartment_id: + raise RuntimeError( + "OCI_COMPARTMENT_ID or OCI_GENAI_PROJECT_OCID is required for LLM_PROVIDER=oci_sdk" + ) + + endpoint = self._normalize_endpoint(effective.get("base_url") or settings.OCI_GENAI_BASE_URL) + if endpoint_id: + serving_mode = models.DedicatedServingMode(endpoint_id=endpoint_id) + else: + serving_mode = models.OnDemandServingMode(model_id=model) + + content = [models.TextContent(text=prompt)] if hasattr(models, "TextContent") else prompt + message = models.UserMessage(content=content) + chat_request = models.GenericChatRequest( + messages=[message], + temperature=effective.get("temperature", settings.LLM_TEMPERATURE), + max_tokens=effective.get("max_tokens", settings.LLM_MAX_TOKENS), + ) + details = models.ChatDetails( + compartment_id=compartment_id, + serving_mode=serving_mode, + chat_request=chat_request, + ) + response = await asyncio.to_thread(self._client(endpoint).chat, details) + return self._extract_answer(response) + + +def create_llm_client() -> LLMClient: + provider = (settings.LLM_PROVIDER or "mock").lower() + if provider in ("mock", "none"): + return MockLLMClient() + if provider in ("oci_openai", "openai_compatible", "oci"): + return OCICompatibleLLMClient() + if provider == "oci_sdk": + return OCISDKLLMClient() + raise ValueError(f"Unsupported LLM_PROVIDER={settings.LLM_PROVIDER}") diff --git a/evaluator/llm/profile_resolver.py b/evaluator/llm/profile_resolver.py new file mode 100644 index 0000000..e6b3ce6 --- /dev/null +++ b/evaluator/llm/profile_resolver.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import copy +import os +import re +from pathlib import Path +from typing import Any + +import yaml + + +def _canonical_profile_name(value: str | None) -> str: + name = (value or "default").strip() + name = name.replace("-", "_").replace(".", "_").replace(" ", "_") + name = re.sub(r"(? Path | None: + root = getattr(self.settings, "project_root", Path.cwd()) + configured = Path(getattr(self.settings, "LLM_PROFILES_PATH", "") or "").expanduser() + candidates = [] + if configured: + candidates.append(configured if configured.is_absolute() else Path(root) / configured) + candidates += [ + Path(root) / "llm_profiles.yaml", + Path(root) / "configs/llm_profiles/llm_profiles.yaml", + Path(root) / "config/llm_profiles.yaml", + Path("llm_profiles.yaml"), + Path("configs/llm_profiles/llm_profiles.yaml"), + ] + for path in candidates: + if path and path.exists() and path.is_file(): + return path + return None + + def _load_profiles(self, path: Path) -> dict[str, dict[str, Any]]: + # Expand ${VAR} placeholders, matching the way env-driven framework config is commonly used. + text = os.path.expandvars(path.read_text(encoding="utf-8")) + data = yaml.safe_load(text) or {} + raw = data.get("profiles", data) + profiles = {} + for name, value in raw.items(): + if isinstance(value, dict): + profiles[_canonical_profile_name(str(name))] = dict(value) + return profiles + + def env_defaults(self) -> dict[str, Any]: + return { + "provider": getattr(self.settings, "LLM_PROVIDER", "mock"), + "model": getattr(self.settings, "OCI_GENAI_MODEL", "mock-llm"), + "temperature": getattr(self.settings, "LLM_TEMPERATURE", 0.0), + "max_tokens": getattr(self.settings, "LLM_MAX_TOKENS", 900), + "timeout_seconds": getattr(self.settings, "LLM_TIMEOUT_SECONDS", 120), + "base_url": getattr(self.settings, "OCI_GENAI_BASE_URL", None), + "api_key": getattr(self.settings, "OCI_GENAI_API_KEY", None), + "project_ocid": getattr(self.settings, "OCI_GENAI_PROJECT_OCID", None), + "endpoint_id": getattr(self.settings, "oci_genai_endpoint_id", None), + "compartment_id": getattr(self.settings, "oci_compartment_id", None), + } + + def resolve(self, profile_name: str | None = None, **overrides) -> dict[str, Any]: + profile_key = _canonical_profile_name(profile_name) + effective = self.env_defaults() + + if self.enabled: + effective.update(copy.deepcopy(self._profiles.get("default") or {})) + effective.update(copy.deepcopy(self._profiles.get(profile_key) or {})) + + for key, value in overrides.items(): + if value is not None: + effective[key] = value + + effective["profile_name"] = profile_key + return effective diff --git a/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc b/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc new file mode 100644 index 0000000..d951230 Binary files /dev/null and b/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc differ diff --git a/evaluator/output/legacy_exporter.py b/evaluator/output/legacy_exporter.py new file mode 100644 index 0000000..0fc268e --- /dev/null +++ b/evaluator/output/legacy_exporter.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import gzip +import json +from collections import defaultdict +from pathlib import Path +from datetime import datetime +from zoneinfo import ZoneInfo +from typing import Any + +from evaluator.config.settings import settings +from evaluator.persistence.repository import EvaluationRepository +from evaluator.analytics.vloop import max_consecutive_intent_streak + +HEADER = [ + "judgeScore", "accuracyScore", "alucinationScore", "promptLength", "loop", + "inferredCsiScore", "resolution", "conversationPrecision", + "uraCallId", "channelId", "sessionId", "messageId" +] + + +def _q(v) -> str: + return '"' + str("" if v is None else v).replace('"', '""') + '"' + + +def _metric(v) -> str | float | int: + # TIM specification: trace metric None must be serialized as "0.00". + return "0.00" if v is None else v + + +def normalize_raw(raw): + if hasattr(raw, "read"): + raw = raw.read() + if isinstance(raw, bytes): + raw = raw.decode("utf-8") + if isinstance(raw, str): + raw = raw.strip() + if not raw: + return {} + raw = json.loads(raw) + if isinstance(raw, str): + raw = json.loads(raw) + return raw if isinstance(raw, dict) else {} + + +def extract_prompt_length(raw: dict) -> int: + """TIM metric: raw character count of trace.input captured by the collector.""" + nested = raw.get("raw") if isinstance(raw.get("raw"), dict) else {} + if "prompt_length" in nested: + try: + return int(nested.get("prompt_length") or 0) + except Exception: + pass + return len(str(raw.get("input_text") or "")) + + +def extract_ura_call_id(raw: dict, metadata: dict | None = None, message_id: str | None = None) -> str: + metadata = metadata or {} + raw_metadata = raw.get("metadata") if isinstance(raw.get("metadata"), dict) else {} + business_context = ( + metadata.get("business_context") + or metadata.get("businessContext") + or raw.get("business_context") + or raw.get("businessContext") + or raw_metadata.get("business_context") + or raw_metadata.get("businessContext") + or {} + ) + if not isinstance(business_context, dict): + business_context = {} + + nested = raw.get("raw") if isinstance(raw.get("raw"), dict) else {} + trace = nested.get("trace") if isinstance(nested.get("trace"), dict) else {} + detail = nested.get("detail") if isinstance(nested.get("detail"), dict) else {} + trace_input = trace.get("input") if isinstance(trace.get("input"), dict) else {} + detail_input = detail.get("input") if isinstance(detail.get("input"), dict) else {} + trace_metadata = trace.get("metadata") if isinstance(trace.get("metadata"), dict) else {} + detail_metadata = detail.get("metadata") if isinstance(detail.get("metadata"), dict) else {} + trace_bc = trace_input.get("business_context") if isinstance(trace_input.get("business_context"), dict) else {} + detail_bc = detail_input.get("business_context") if isinstance(detail_input.get("business_context"), dict) else {} + + return str( + business_context.get("interaction_key") + or business_context.get("ura_call_id") + or metadata.get("ura_call_id") + or metadata.get("uraCallId") + or metadata.get("interaction_key") + or trace_metadata.get("ura_call_id") + or detail_metadata.get("ura_call_id") + or trace_bc.get("interaction_key") + or detail_bc.get("interaction_key") + or message_id + or "" + ) + + +def _session_loop_map(rows) -> dict[str, int]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + session_id = row[1] + raw_json = row[8] + try: + grouped[str(session_id)].append(normalize_raw(raw_json)) + except Exception: + grouped[str(session_id)].append({}) + return {sid: max_consecutive_intent_streak(items) for sid, items in grouped.items()} + + +def export_legacy_txt_gz(repo: EvaluationRepository, run_id: str, agent_id: str) -> Path: + output_dir = settings.path(settings.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + now_brt = datetime.now(ZoneInfo("America/Sao_Paulo")) + path = output_dir / f"AGENTE_{agent_id}_LLM_JUDGE_{now_brt.strftime('%Y%m%d_%H%M%S')}.TXT.GZ" + + with repo.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select SESSION_ID, INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION + from {repo.store.t('EVALUATION_RESULT')} + where RUN_ID = :run_id and JUDGE_TYPE = 'SESSION' + """, {"run_id": run_id}) + session_metrics = { + sid: {"inferredCsiScore": csi, "resolution": res, "conversationPrecision": prec} + for sid, csi, res, prec in cur.fetchall() + } + + cur.execute(f""" + select r.TRACE_ID, r.SESSION_ID, r.JUDGE_SCORE, r.ACCURACY_SCORE, + r.ALUCINATION_SCORE, r.RATIONALE, i.CHANNEL, i.MESSAGE_ID, i.RAW_JSON + from {repo.store.t('EVALUATION_RESULT')} r + left join {repo.store.t('EVALUATION_ITEM')} i on i.ITEM_ID = r.ITEM_ID + where r.RUN_ID = :run_id and r.JUDGE_TYPE = 'TRACE' + order by r.CREATED_AT + """, {"run_id": run_id}) + rows = cur.fetchall() + + loop_by_session = _session_loop_map(rows) + + with gzip.open(path, "wt", encoding="utf-8") as f: + f.write("|;".join(_q(v) for v in HEADER) + "\n") + for trace_id, session_id, judge, accuracy, alucination, rationale, channel, message_id, raw_json in rows: + session = session_metrics.get(session_id, {}) + raw = {} + ura_call_id = "" + channel_id = channel or "" + prompt_length = 0 + try: + raw = normalize_raw(raw_json) + metadata = raw.get("metadata") if isinstance(raw.get("metadata"), dict) else {} + channel_id = ( + metadata.get("channel_id") + or metadata.get("channelId") + or metadata.get("channel") + or channel_id + ) + ura_call_id = extract_ura_call_id(raw, metadata, message_id) + prompt_length = extract_prompt_length(raw) + except Exception as exc: + print(f"[legacy_exporter] metadata extraction failed trace_id={trace_id}: {exc}") + + vals = [ + _metric(judge), + _metric(accuracy), + _metric(alucination), + prompt_length, + loop_by_session.get(str(session_id), 0), + session.get("inferredCsiScore"), + session.get("resolution"), + session.get("conversationPrecision"), + ura_call_id, + channel_id, + session_id, + message_id or trace_id, + ] + f.write("|;".join(_q(v) for v in vals) + "\n") + f.write("|;".join([_q("TOTAL"), _q(len(rows))]) + "\n") + return path diff --git a/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc b/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc new file mode 100644 index 0000000..9b43ef7 Binary files /dev/null and b/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc differ diff --git a/evaluator/persistence/__pycache__/repository.cpython-313.pyc b/evaluator/persistence/__pycache__/repository.cpython-313.pyc new file mode 100644 index 0000000..038f9c1 Binary files /dev/null and b/evaluator/persistence/__pycache__/repository.cpython-313.pyc differ diff --git a/evaluator/persistence/oracle_store.py b/evaluator/persistence/oracle_store.py new file mode 100644 index 0000000..fa2d97c --- /dev/null +++ b/evaluator/persistence/oracle_store.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import asyncio +import json +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any + + +def _json_dumps(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _json_loads(value: str | bytes | None, default: Any): + if value is None: + return default + if isinstance(value, bytes): + value = value.decode("utf-8") + try: + return json.loads(value) + except Exception: + return default + + +@dataclass +class OracleSettings: + user: str + password: str + dsn: str + wallet_location: str | None = None + wallet_password: str | None = None + table_prefix: str = "AGENTFW" + + +class OracleStore: + """Oracle Autonomous Database store following the Agent Framework pattern. + + Uses direct oracledb.connect() with wallet arguments. Synchronous DB calls are + exposed through asyncio.to_thread(), matching the main framework style. + """ + + def __init__(self, settings, auto_init_schema: bool = False): + self.settings = settings + self.cfg = OracleSettings( + user=settings.ADB_USER or "", + password=settings.ADB_PASSWORD or "", + dsn=settings.ADB_DSN or "", + wallet_location=getattr(settings, "ADB_WALLET_LOCATION", None), + wallet_password=getattr(settings, "ADB_WALLET_PASSWORD", None), + table_prefix=(getattr(settings, "ADB_TABLE_PREFIX", "AGENTFW") or "AGENTFW").upper().rstrip("_"), + ) + if not self.cfg.user or not self.cfg.password or not self.cfg.dsn: + raise RuntimeError("ADB_USER, ADB_PASSWORD and ADB_DSN are required") + if auto_init_schema: + self._init_schema() + + @staticmethod + def now() -> datetime: + return datetime.now(timezone.utc) + + def t(self, name: str) -> str: + return f"{self.cfg.table_prefix}_{name}".upper() + + @contextmanager + def connect(self): + import oracledb + oracledb.defaults.fetch_lobs = False + kwargs = {} + if self.cfg.wallet_location: + kwargs["config_dir"] = self.cfg.wallet_location + kwargs["wallet_location"] = self.cfg.wallet_location + if self.cfg.wallet_password: + kwargs["wallet_password"] = self.cfg.wallet_password + conn = oracledb.connect(user=self.cfg.user, password=self.cfg.password, dsn=self.cfg.dsn, **kwargs) + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + async def to_thread(self, func, *args, **kwargs): + return await asyncio.to_thread(func, *args, **kwargs) + + def _exec_ddl_ignore_exists(self, cur, ddl: str): + try: + cur.execute(ddl) + except Exception as exc: + msg = str(exc) + if "ORA-00955" in msg or "ORA-01408" in msg or "ORA-02275" in msg: + return + raise + + def _column_exists(self, cur, table_name: str, column_name: str) -> bool: + cur.execute(""" + select count(*) + from user_tab_columns + where table_name = :table_name + and column_name = :column_name + """, {"table_name": self.t(table_name), "column_name": column_name.upper()}) + return int(cur.fetchone()[0] or 0) > 0 + + def _ensure_column(self, cur, table_name: str, column_name: str, ddl_type: str): + if not self._column_exists(cur, table_name, column_name): + cur.execute(f"alter table {self.t(table_name)} add {column_name.upper()} {ddl_type}") + + def drop_schema(self): + tables = [ + "EVALUATION_RESULT", + "EVALUATION_FINDING", + "EVALUATION_METRIC", + "EVALUATION_ITEM", + "EVALUATION_PROGRESS_EVENT", + "EVALUATION_RUN", + ] + with self.connect() as conn: + cur = conn.cursor() + for table in tables: + try: + cur.execute(f"drop table {self.t(table)} cascade constraints purge") + except Exception as exc: + if "ORA-00942" in str(exc): + continue + raise + + def _init_schema(self): + with self.connect() as conn: + cur = conn.cursor() + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_RUN')} ( + RUN_ID varchar2(64) primary key, + AGENT_ID varchar2(128), + SOURCE varchar2(64), + PERIOD_START timestamp with time zone, + PERIOD_END timestamp with time zone, + STATUS varchar2(32) not null, + TOTAL_ITEMS number default 0 not null, + PROCESSED_ITEMS number default 0 not null, + FAILED_ITEMS number default 0 not null, + RETRY_COUNT number default 0 not null, + ERROR_MESSAGE clob, + LAST_HEARTBEAT_AT timestamp with time zone, + CREATED_AT timestamp with time zone not null, + UPDATED_AT timestamp with time zone not null + ) + """) + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_PROGRESS_EVENT')} ( + ID number generated always as identity primary key, + RUN_ID varchar2(64) not null, + STAGE varchar2(128) not null, + MESSAGE varchar2(1000), + DETAILS_JSON clob check (DETAILS_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_PROGRESS_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_PROGRESS_RUN')} on {self.t('EVALUATION_PROGRESS_EVENT')}(RUN_ID, CREATED_AT)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_ITEM')} ( + ITEM_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + TRACE_ID varchar2(256), + SESSION_ID varchar2(256), + MESSAGE_ID varchar2(256), + AGENT_ID varchar2(128), + CHANNEL varchar2(64), + STATUS varchar2(32) not null, + ATTEMPT_COUNT number default 0 not null, + ERROR_MESSAGE clob, + RAW_JSON clob check (RAW_JSON is json), + CREATED_AT timestamp with time zone not null, + UPDATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_ITEM_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_ITEM_RUN')} on {self.t('EVALUATION_ITEM')}(RUN_ID, STATUS, CREATED_AT)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_RESULT')} ( + RESULT_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + TRACE_ID varchar2(256), + SESSION_ID varchar2(256), + AGENT_ID varchar2(128), + JUDGE_NAME varchar2(128) not null, + JUDGE_TYPE varchar2(32), + SCORE number, + JUDGE_SCORE number, + ACCURACY_SCORE number, + ALUCINATION_SCORE number, + HALLUCINATION_SCORE number, + INFERRED_CSI_SCORE number, + RESOLUTION number, + CONVERSATION_PRECISION number, + TOOL_USAGE_SCORE number, + ROUTING_SCORE number, + RATIONALE clob, + REASONING clob, + RESULT_JSON clob check (RESULT_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_RESULT_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID), + constraint {self.t('FK_EVAL_RESULT_ITEM')} + foreign key (ITEM_ID) references {self.t('EVALUATION_ITEM')}(ITEM_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_RESULT_RUN')} on {self.t('EVALUATION_RESULT')}(RUN_ID, ITEM_ID)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_METRIC')} ( + METRIC_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + METRIC_NAME varchar2(128) not null, + METRIC_VALUE number, + DIMENSIONS_JSON clob check (DIMENSIONS_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_METRIC_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_FINDING')} ( + FINDING_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + SEVERITY varchar2(32), + CATEGORY varchar2(128), + TITLE varchar2(512), + DESCRIPTION clob, + EVIDENCE_JSON clob check (EVIDENCE_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_FINDING_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + + # Non-destructive compatibility for older generated schemas. + for col, typ in [ + ("RETRY_COUNT", "number default 0"), + ("ERROR_MESSAGE", "clob"), + ("LAST_HEARTBEAT_AT", "timestamp with time zone"), + ("UPDATED_AT", "timestamp with time zone"), + ]: + self._ensure_column(cur, "EVALUATION_RUN", col, typ) + for col, typ in [ + ("ID", "number generated always as identity"), + ]: + # Identity column cannot always be added cleanly; ignore if table is old without ID. + try: + self._ensure_column(cur, "EVALUATION_PROGRESS_EVENT", col, typ) + except Exception: + pass + for col, typ in [ + ("JUDGE_NAME", "varchar2(128) default 'unknown_judge' not null"), + ("JUDGE_TYPE", "varchar2(32)"), + ("SCORE", "number"), + ("JUDGE_SCORE", "number"), + ("ACCURACY_SCORE", "number"), + ("ALUCINATION_SCORE", "number"), + ("HALLUCINATION_SCORE", "number"), + ("INFERRED_CSI_SCORE", "number"), + ("RESOLUTION", "number"), + ("CONVERSATION_PRECISION", "number"), + ("TOOL_USAGE_SCORE", "number"), + ("ROUTING_SCORE", "number"), + ("RATIONALE", "clob"), + ("REASONING", "clob"), + ("RESULT_JSON", "clob"), + ]: + self._ensure_column(cur, "EVALUATION_RESULT", col, typ) diff --git a/evaluator/persistence/repository.py b/evaluator/persistence/repository.py new file mode 100644 index 0000000..5fbacca --- /dev/null +++ b/evaluator/persistence/repository.py @@ -0,0 +1,411 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, ItemStatus, RunStatus, TraceJudgeResult, SessionJudgeResult +from evaluator.persistence.oracle_store import OracleStore, _json_dumps, _json_loads +from evaluator.time_utils import brt_naive + + +class EvaluationRepository: + def __init__(self, auto_init_schema: bool = False): + self.store = OracleStore(settings, auto_init_schema=auto_init_schema) + + def create_run(self, period_start: datetime, period_end: datetime, source: str, agent_id: str | None = None) -> str: + run_id = str(uuid.uuid4()) + now = self.store.now() + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RUN')} + (RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS, TOTAL_ITEMS, + PROCESSED_ITEMS, FAILED_ITEMS, RETRY_COUNT, LAST_HEARTBEAT_AT, CREATED_AT, UPDATED_AT) + values (:run_id, :agent_id, :period_start, :period_end, :source, :status, + 0, 0, 0, 0, :heartbeat_at, :created_at, :updated_at) + """, { + "run_id": run_id, + "agent_id": agent_id, + "period_start": brt_naive(period_start), + "period_end": brt_naive(period_end), + "source": source, + "status": RunStatus.RUNNING.value, + "heartbeat_at": now, + "created_at": now, + "updated_at": now, + }) + return run_id + + async def acreate_run(self, *args, **kwargs): + return await self.store.to_thread(self.create_run, *args, **kwargs) + + def record_progress(self, run_id: str, stage: str, message: str = "", details: dict | None = None): + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_PROGRESS_EVENT')} + (RUN_ID, STAGE, MESSAGE, DETAILS_JSON, CREATED_AT) + values (:run_id, :stage, :message, :details_json, :created_at) + """, { + "run_id": run_id, + "stage": stage, + "message": (message or "")[:1000], + "details_json": _json_dumps(details or {}), + "created_at": self.store.now(), + }) + + async def arecord_progress(self, *args, **kwargs): + return await self.store.to_thread(self.record_progress, *args, **kwargs) + + def insert_items(self, run_id: str, records: list[ConversationRecord]) -> int: + inserted = 0 + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + for record in records: + try: + cur.execute(f""" + insert into {self.store.t('EVALUATION_ITEM')} + (ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL, + STATUS, ATTEMPT_COUNT, RAW_JSON, CREATED_AT, UPDATED_AT) + values (:item_id, :run_id, :trace_id, :session_id, :message_id, :agent_id, + :channel, :status, 0, :raw_json, :created_at, :updated_at) + """, { + "item_id": str(uuid.uuid4()), + "run_id": run_id, + "trace_id": record.trace_id, + "session_id": record.session_id, + "message_id": record.message_id, + "agent_id": record.agent_id, + "channel": record.channel, + "status": ItemStatus.PENDING.value, + "raw_json": record.model_dump_json(), + "created_at": now, + "updated_at": now, + }) + inserted += 1 + except Exception as exc: + if "ORA-00001" not in str(exc): + raise + cur.execute(f""" + update {self.store.t('EVALUATION_RUN')} + set TOTAL_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id + ), + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, {"run_id": run_id, "updated_at": self.store.now()}) + return inserted + + async def ainsert_items(self, *args, **kwargs): + return await self.store.to_thread(self.insert_items, *args, **kwargs) + + def fetch_next_items(self, run_id: str, batch_size: int) -> list[dict]: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL, + STATUS, ATTEMPT_COUNT, RAW_JSON + from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id + and STATUS in (:pending, :failed) + and ATTEMPT_COUNT < :max_attempts + order by CREATED_AT + ) where rownum <= :batch_size + """, { + "run_id": run_id, + "pending": ItemStatus.PENDING.value, + "failed": ItemStatus.FAILED.value, + "max_attempts": settings.max_attempts, + "batch_size": batch_size, + }) + cols = [d[0].lower() for d in cur.description] + return [dict(zip(cols, row)) for row in cur.fetchall()] + + async def afetch_next_items(self, *args, **kwargs): + return await self.store.to_thread(self.fetch_next_items, *args, **kwargs) + + def mark_item_processing(self, item_id: str): + with self.store.connect() as conn: + conn.cursor().execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + ATTEMPT_COUNT = ATTEMPT_COUNT + 1, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.PROCESSING.value, + "updated_at": self.store.now(), + "item_id": item_id, + }) + + async def amark_item_processing(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_processing, *args, **kwargs) + + def mark_item_completed(self, run_id: str, item_id: str): + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.COMPLETED.value, + "updated_at": now, + "item_id": item_id, + }) + self._refresh_run_counters(cur, run_id, now) + + async def amark_item_completed(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_completed, *args, **kwargs) + + def mark_item_failed(self, run_id: str, item_id: str, error: str): + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + ERROR_MESSAGE = :error, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.FAILED.value, + "error": (error or "")[:4000], + "updated_at": now, + "item_id": item_id, + }) + self._refresh_run_counters(cur, run_id, now) + + async def amark_item_failed(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_failed, *args, **kwargs) + + def _refresh_run_counters(self, cur, run_id: str, updated_at): + cur.execute(f""" + update {self.store.t('EVALUATION_RUN')} + set PROCESSED_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id and STATUS = :completed + ), + FAILED_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id and STATUS = :failed + ), + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, { + "run_id": run_id, + "completed": ItemStatus.COMPLETED.value, + "failed": ItemStatus.FAILED.value, + "updated_at": updated_at, + }) + + def save_trace_result(self, run_id: str, item_id: str, record: ConversationRecord, result: TraceJudgeResult): + judge_name = getattr(result, "judge_name", None) or "trace_metrics" + judge_type = (getattr(result, "judge_type", None) or "TRACE").upper() + score = getattr(result, "judgeScore", None) + accuracy = getattr(result, "accuracyScore", None) + alucination = getattr(result, "alucinationScore", None) + rationale = getattr(result, "rationale", None) or "" + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RESULT')} + (RESULT_ID, RUN_ID, ITEM_ID, TRACE_ID, SESSION_ID, AGENT_ID, JUDGE_NAME, + JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE, ALUCINATION_SCORE, + RATIONALE, RESULT_JSON, CREATED_AT) + values (:result_id, :run_id, :item_id, :trace_id, :session_id, :agent_id, + :judge_name, :judge_type, :score, :judge_score, :accuracy_score, + :alucination_score, :rationale, :result_json, :created_at) + """, { + "result_id": str(uuid.uuid4()), + "run_id": run_id, + "item_id": item_id, + "trace_id": record.trace_id, + "session_id": record.session_id, + "agent_id": record.agent_id, + "judge_name": judge_name, + "judge_type": judge_type, + "score": score, + "judge_score": score, + "accuracy_score": accuracy, + "alucination_score": alucination, + "rationale": rationale, + "result_json": result.model_dump_json(), + "created_at": self.store.now(), + }) + + async def asave_trace_result(self, *args, **kwargs): + return await self.store.to_thread(self.save_trace_result, *args, **kwargs) + + def save_session_result(self, run_id: str, session_id: str, agent_id: str | None, result: SessionJudgeResult): + judge_name = getattr(result, "judge_name", None) or "session_metrics" + judge_type = (getattr(result, "judge_type", None) or "SESSION").upper() + rationale = getattr(result, "rationale", None) or "" + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RESULT')} + (RESULT_ID, RUN_ID, SESSION_ID, AGENT_ID, JUDGE_NAME, JUDGE_TYPE, + INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION, RATIONALE, + RESULT_JSON, CREATED_AT) + values (:result_id, :run_id, :session_id, :agent_id, :judge_name, :judge_type, + :csi, :resolution, :precision, :rationale, :result_json, :created_at) + """, { + "result_id": str(uuid.uuid4()), + "run_id": run_id, + "session_id": session_id, + "agent_id": agent_id, + "judge_name": judge_name, + "judge_type": judge_type, + "csi": getattr(result, "inferredCsiScore", None), + "resolution": getattr(result, "resolution", None), + "precision": getattr(result, "conversationPrecision", None), + "rationale": rationale, + "result_json": result.model_dump_json(), + "created_at": self.store.now(), + }) + + async def asave_session_result(self, *args, **kwargs): + return await self.store.to_thread(self.save_session_result, *args, **kwargs) + + def mark_run_status(self, run_id: str, status: RunStatus, error: str | None = None): + with self.store.connect() as conn: + conn.cursor().execute(f""" + update {self.store.t('EVALUATION_RUN')} + set STATUS = :status, + ERROR_MESSAGE = :error, + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, { + "status": status.value, + "error": error, + "updated_at": self.store.now(), + "run_id": run_id, + }) + + async def amark_run_status(self, *args, **kwargs): + return await self.store.to_thread(self.mark_run_status, *args, **kwargs) + + def summarize_run(self, run_id: str) -> dict: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id), + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'COMPLETED'), + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'FAILED'), + (select count(*) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE'), + (select avg(JUDGE_SCORE) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE') + from dual + """, {"run_id": run_id}) + r = cur.fetchone() + return { + "run_id": run_id, + "total_items": int(r[0] or 0), + "completed_items": int(r[1] or 0), + "failed_items": int(r[2] or 0), + "evaluations": int(r[3] or 0), + "avg_score": float(r[4]) if r[4] is not None else None, + } + + async def asummarize_run(self, *args, **kwargs): + return await self.store.to_thread(self.summarize_run, *args, **kwargs) + + def get_run_progress(self, run_id: str, event_limit: int = 20) -> dict: + summary = self.summarize_run(run_id) + total = summary["total_items"] or 0 + done = summary["completed_items"] + summary["failed_items"] + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select STAGE, MESSAGE, DETAILS_JSON, CREATED_AT + from {self.store.t('EVALUATION_PROGRESS_EVENT')} + where RUN_ID = :run_id + order by CREATED_AT desc + ) where rownum <= :max_rows + """, {"run_id": run_id, "max_rows": event_limit}) + events = [ + { + "stage": s, + "message": m, + "details": _json_loads(d.read() if hasattr(d, "read") else d, {}), + "created_at": str(c), + } + for s, m, d, c in cur.fetchall() + ] + return { + **summary, + "done_items": done, + "percent_complete": round((done / total) * 100, 2) if total else 0.0, + "events": events, + } + + async def aget_run_progress(self, *args, **kwargs): + return await self.store.to_thread(self.get_run_progress, *args, **kwargs) + + def list_runs(self, limit: int = 20): + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS, + TOTAL_ITEMS, PROCESSED_ITEMS, FAILED_ITEMS, CREATED_AT, UPDATED_AT + from {self.store.t('EVALUATION_RUN')} + order by CREATED_AT desc + ) where rownum <= :max_rows + """, {"max_rows": limit}) + return [ + { + "run_id": r[0], + "agent_id": r[1], + "period_start": str(r[2]), + "period_end": str(r[3]), + "source": r[4], + "status": r[5], + "total_items": int(r[6] or 0), + "processed_items": int(r[7] or 0), + "failed_items": int(r[8] or 0), + "created_at": str(r[9]), + "updated_at": str(r[10]), + } + for r in cur.fetchall() + ] + + async def alist_runs(self, *args, **kwargs): + return await self.store.to_thread(self.list_runs, *args, **kwargs) + + def list_results(self, run_id: str, limit: int = 100) -> list[dict]: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select JUDGE_NAME, JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE, + ALUCINATION_SCORE, INFERRED_CSI_SCORE, RESOLUTION, + CONVERSATION_PRECISION, RATIONALE, TRACE_ID, SESSION_ID, CREATED_AT + from {self.store.t('EVALUATION_RESULT')} + where RUN_ID = :run_id + order by CREATED_AT desc + """, {"run_id": run_id}) + return [ + { + "judge_name": r[0], + "judge_type": r[1], + "score": r[2], + "judge_score": r[3], + "accuracy_score": r[4], + "alucination_score": r[5], + "inferred_csi_score": r[6], + "resolution": r[7], + "conversation_precision": r[8], + "rationale": r[9], + "trace_id": r[10], + "session_id": r[11], + "created_at": str(r[12]), + } + for r in cur.fetchall()[:limit] + ] + + async def alist_results(self, *args, **kwargs): + return await self.store.to_thread(self.list_results, *args, **kwargs) diff --git a/evaluator/prompts/__pycache__/loader.cpython-313.pyc b/evaluator/prompts/__pycache__/loader.cpython-313.pyc new file mode 100644 index 0000000..9ae4608 Binary files /dev/null and b/evaluator/prompts/__pycache__/loader.cpython-313.pyc differ diff --git a/evaluator/prompts/loader.py b/evaluator/prompts/loader.py new file mode 100644 index 0000000..5e800f7 --- /dev/null +++ b/evaluator/prompts/loader.py @@ -0,0 +1,11 @@ +from __future__ import annotations +import yaml +from evaluator.config.settings import settings + + +def load_prompt(path: str, key: str) -> str: + p = settings.path(path) + data = yaml.safe_load(p.read_text()) or {} + if key not in data: + raise KeyError(f"Prompt key {key!r} not found in {p}") + return str(data[key]) diff --git a/evaluator/publishers/__pycache__/gcs_export.cpython-313.pyc b/evaluator/publishers/__pycache__/gcs_export.cpython-313.pyc new file mode 100644 index 0000000..f277e17 Binary files /dev/null and b/evaluator/publishers/__pycache__/gcs_export.cpython-313.pyc differ diff --git a/evaluator/publishers/__pycache__/langfuse_job_audit.cpython-313.pyc b/evaluator/publishers/__pycache__/langfuse_job_audit.cpython-313.pyc new file mode 100644 index 0000000..71848df Binary files /dev/null and b/evaluator/publishers/__pycache__/langfuse_job_audit.cpython-313.pyc differ diff --git a/evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc b/evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc new file mode 100644 index 0000000..43b4c9e Binary files /dev/null and b/evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc differ diff --git a/evaluator/publishers/gcs_export.py b/evaluator/publishers/gcs_export.py new file mode 100644 index 0000000..05da2b0 --- /dev/null +++ b/evaluator/publishers/gcs_export.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from pathlib import Path +import yaml + +from evaluator.config.settings import settings + + +def _load_bucket_mapping() -> dict[str, str]: + config_path = settings.path(settings.storage_config_path) + if not config_path.exists(): + raise RuntimeError( + f"Storage configuration file not found: {config_path}. " + "Set STORAGE_CONFIG_PATH to a YAML file containing storage.buckets." + ) + data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + storage = data.get("storage", {}) if isinstance(data, dict) else {} + buckets = storage.get("buckets", {}) if isinstance(storage, dict) else {} + if not isinstance(buckets, dict): + raise RuntimeError(f"Invalid storage.buckets mapping in {config_path}") + return {str(k).lower(): str(v) for k, v in buckets.items() if v} + + +def resolve_gcs_bucket() -> str: + if settings.judge_gcs_bucket: + return settings.judge_gcs_bucket + env = (settings.app_env or "").lower() + bucket = _load_bucket_mapping().get(env) + if not bucket: + raise RuntimeError( + f"No GCS bucket configured for APP_ENV={settings.app_env!r}. " + "Configure storage.buckets in STORAGE_CONFIG_PATH or set JUDGE_GCS_BUCKET." + ) + return bucket + + +def upload_evaluation_file(path: Path, prefix: str = "") -> str: + if not settings.enable_gcs_upload: + raise RuntimeError("GCS upload requested while ENABLE_GCS_UPLOAD=false") + + from google.cloud import storage + + bucket_name = resolve_gcs_bucket() + object_name = "/".join( + part.strip("/") for part in (prefix, path.name) if part and part.strip("/") + ) + + client = storage.Client() + bucket = client.bucket(bucket_name) + blob = bucket.blob(object_name) + blob.upload_from_filename(str(path), content_type="application/gzip") + return f"gs://{bucket_name}/{object_name}" diff --git a/evaluator/publishers/langfuse_job_audit.py b/evaluator/publishers/langfuse_job_audit.py new file mode 100644 index 0000000..a2b492a --- /dev/null +++ b/evaluator/publishers/langfuse_job_audit.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from typing import Any + +import httpx + +from evaluator.collectors.langfuse_credentials import resolve_langfuse_credentials +from evaluator.config.settings import settings + + +class LangfuseJobAuditPublisher: + """Audit each agent batch as judge-job-{agentId} in that agent's Langfuse project.""" + + def new_trace_id(self) -> str: + return uuid.uuid4().hex + + async def publish( + self, + trace_id: str, + agent_id: str, + status: str, + *, + period_start: datetime | None = None, + period_end: datetime | None = None, + details: dict[str, Any] | None = None, + ) -> dict[str, Any] | None: + if not settings.enable_langfuse: + return None + creds = resolve_langfuse_credentials(agent_id) + event_timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + body: dict[str, Any] = { + "id": trace_id, + "timestamp": event_timestamp, + "name": f"judge-job-{agent_id}", + "metadata": {"status": status, "agentId": agent_id, **(details or {})}, + } + if period_start or period_end: + body["input"] = { + "period_start": period_start.isoformat() if period_start else None, + "period_end": period_end.isoformat() if period_end else None, + } + if status in {"completed", "failed"}: + body["output"] = details or {"status": status} + payload = { + "batch": [{"id": str(uuid.uuid4()), "type": "trace-create", "timestamp": event_timestamp, "body": body}] + } + async with httpx.AsyncClient(base_url=creds.host, timeout=30) as client: + resp = await client.post("/api/public/ingestion", json=payload, auth=creds.auth) + if resp.status_code >= 400: + raise RuntimeError(f"Langfuse audit failed {resp.status_code}: {resp.text}") + return {"ok": True, "trace_id": trace_id, "status": status, "credential_source": creds.source} diff --git a/evaluator/publishers/langfuse_scores.py b/evaluator/publishers/langfuse_scores.py new file mode 100644 index 0000000..0df91f7 --- /dev/null +++ b/evaluator/publishers/langfuse_scores.py @@ -0,0 +1,23 @@ +from __future__ import annotations +import httpx +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, TraceJudgeResult + +class LangfuseScorePublisher: + async def publish_trace_score(self, record: ConversationRecord, result: TraceJudgeResult): + if not settings.can_publish_langfuse_scores or not record.trace_id: + return None + auth = (settings.langfuse_public_key, settings.langfuse_secret_key) + payloads = [ + {'traceId': record.trace_id, 'name': 'offline_judge_score', 'value': result.judgeScore, 'comment': result.rationale}, + {'traceId': record.trace_id, 'name': 'offline_accuracy_score', 'value': result.accuracyScore, 'comment': result.rationale}, + {'traceId': record.trace_id, 'name': 'offline_alucination_score', 'value': result.alucinationScore, 'comment': result.rationale}, + ] + payloads = [payload for payload in payloads if payload['value'] is not None] + async with httpx.AsyncClient(base_url=settings.langfuse_host, timeout=30) as client: + for payload in payloads: + resp = await client.post('/api/public/scores', json=payload, auth=auth) + if resp.status_code >= 400: + # Don't fail the run because score publishing is supplementary. + return {'ok': False, 'status': resp.status_code, 'body': resp.text} + return {'ok': True} diff --git a/evaluator/time_utils.py b/evaluator/time_utils.py new file mode 100644 index 0000000..18a0023 --- /dev/null +++ b/evaluator/time_utils.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from datetime import date, datetime, time, timezone +from zoneinfo import ZoneInfo + +BRT = ZoneInfo("America/Sao_Paulo") + + +def as_brt(value: datetime) -> datetime: + """Interpret naive datetimes as America/Sao_Paulo and normalize aware values to BRT.""" + if value.tzinfo is None: + return value.replace(tzinfo=BRT) + return value.astimezone(BRT) + + +def to_utc_z(value: datetime) -> str: + return as_brt(value).astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def brt_day_window(reference_date: date) -> tuple[datetime, datetime]: + start = datetime.combine(reference_date, time(0, 0, 0), tzinfo=BRT) + end = datetime.combine(reference_date, time(23, 59, 59, 999999), tzinfo=BRT) + return start, end + + +def brt_naive(value: datetime) -> datetime: + """Oracle TIMESTAMP storage helper: persist the wall clock in BRT without tzinfo.""" + return as_brt(value).replace(tzinfo=None) diff --git a/evaluator_monitor/__init__.py b/evaluator_monitor/__init__.py new file mode 100644 index 0000000..abeae50 --- /dev/null +++ b/evaluator_monitor/__init__.py @@ -0,0 +1 @@ +"""Read-only web monitor for Agent Framework OCI Evaluator runs.""" diff --git a/evaluator_monitor/__pycache__/__init__.cpython-313.pyc b/evaluator_monitor/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..a036299 Binary files /dev/null and b/evaluator_monitor/__pycache__/__init__.cpython-313.pyc differ diff --git a/evaluator_monitor/__pycache__/app.cpython-313.pyc b/evaluator_monitor/__pycache__/app.cpython-313.pyc new file mode 100644 index 0000000..c71a80b Binary files /dev/null and b/evaluator_monitor/__pycache__/app.cpython-313.pyc differ diff --git a/evaluator_monitor/app.py b/evaluator_monitor/app.py new file mode 100644 index 0000000..7467029 --- /dev/null +++ b/evaluator_monitor/app.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import asyncio +from html import escape + +from fastapi import FastAPI, HTTPException, Query +from fastapi.responses import HTMLResponse, RedirectResponse + +from evaluator.persistence.repository import EvaluationRepository + +app = FastAPI( + title="Agent Framework Evaluator Monitor", + version="1.0.0", + description=( + "Read-only monitoring API/UI for evaluator runs. This component is intentionally " + "separate from the batch evaluator CronJob." + ), +) + + +def _repo() -> EvaluationRepository: + return EvaluationRepository(auto_init_schema=False) + + +@app.get("/health") +async def health() -> dict: + return {"status": "UP", "component": "evaluator-monitor", "read_only": True} + + +@app.get("/api/runs") +async def list_runs(limit: int = Query(50, ge=1, le=500)) -> list[dict]: + return await _repo().alist_runs(limit) + + +@app.get("/api/runs/{run_id}") +async def run_progress(run_id: str, events: int = Query(100, ge=1, le=1000)) -> dict: + try: + data = await _repo().aget_run_progress(run_id, event_limit=events) + except Exception as exc: + if "no data" in str(exc).lower() or "not found" in str(exc).lower(): + raise HTTPException(status_code=404, detail=f"Run {run_id} not found") from exc + raise + return data + + +@app.get("/api/runs/{run_id}/results") +async def run_results(run_id: str, limit: int = Query(100, ge=1, le=1000)) -> list[dict]: + return await _repo().alist_results(run_id, limit) + + +@app.get("/", include_in_schema=False) +async def root(): + return RedirectResponse(url="/ui") + + +@app.get("/ui", response_class=HTMLResponse, include_in_schema=False) +async def ui() -> str: + return """ + + + + + Evaluator Monitor + + + +

Agent Framework Evaluator Monitor

Read-only UI · evaluator CronJob remains a separate batch component
+
+
+ + + Loading… +
+
+

Recent runs

CreatedAgentStatusProgressRun ID
+

Selected run

Select a run to inspect its progress.
+
+

Results

TypeSessionTraceJudgeAccuracyHallucinationCSIResolutionPrecision
Select a run.
+
+ +""" + + +if __name__ == "__main__": + import uvicorn + uvicorn.run("evaluator_monitor.app:app", host="0.0.0.0", port=8080) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..13de2b8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "agent-framework-evaluator" +version = "0.2.0" +description = "Offline LLM-as-a-Judge evaluator for Agent Framework conversations" +requires-python = ">=3.11" +dependencies = [ + "python-dotenv>=1.0.1", + "pydantic>=2.7.0", + "pydantic-settings>=2.2.1", + "oracledb>=2.4.0", + "httpx>=0.27.0", + "openai>=1.40.0", + "oci>=2.111.0", + "google-cloud-storage>=2.18.0", + "pyyaml>=6.0.1", + "typer>=0.12.3", + "click>=8.1.7", + "rich>=13.7.0", + "fastapi>=0.115.0", + "uvicorn>=0.30.0" +] + +[project.scripts] +af-evaluator = "evaluator.cli:app" + +[tool.setuptools.packages.find] +where = ["."] +include = ["evaluator*", "evaluator_monitor*"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__pycache__/__init__.cpython-313.pyc b/tests/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000..d36d839 Binary files /dev/null and b/tests/__pycache__/__init__.cpython-313.pyc differ diff --git a/tests/__pycache__/test_full_tim_compliance.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_full_tim_compliance.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000..be28409 Binary files /dev/null and b/tests/__pycache__/test_full_tim_compliance.cpython-313-pytest-9.0.2.pyc differ diff --git a/tests/__pycache__/test_monitor_and_run_id.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_monitor_and_run_id.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000..185866f Binary files /dev/null and b/tests/__pycache__/test_monitor_and_run_id.cpython-313-pytest-9.0.2.pyc differ diff --git a/tests/__pycache__/test_no_product_hardcodes.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_no_product_hardcodes.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000..2f4dacc Binary files /dev/null and b/tests/__pycache__/test_no_product_hardcodes.cpython-313-pytest-9.0.2.pyc differ diff --git a/tests/__pycache__/test_tim_spec.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_tim_spec.cpython-313-pytest-9.0.2.pyc new file mode 100644 index 0000000..0f09f68 Binary files /dev/null and b/tests/__pycache__/test_tim_spec.cpython-313-pytest-9.0.2.pyc differ diff --git a/tests/test_full_tim_compliance.py b/tests/test_full_tim_compliance.py new file mode 100644 index 0000000..f951366 --- /dev/null +++ b/tests/test_full_tim_compliance.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import asyncio +import os +import tempfile +import unittest +from datetime import datetime +from pathlib import Path +from unittest.mock import patch + +from evaluator.collectors.langfuse import LangfuseCollector +from evaluator.collectors.langfuse_credentials import resolve_langfuse_credentials +from evaluator.config.agents import load_agents +from evaluator.output.legacy_exporter import extract_prompt_length + + +class _Response: + def __init__(self, payload, status_code=200): + self._payload = payload + self.status_code = status_code + self.text = "" + def json(self): + return self._payload + + +class _Client: + def __init__(self, pages): + self.pages = pages + self.calls = [] + async def get(self, path, params=None, auth=None): + self.calls.append((path, dict(params or {}))) + page = int((params or {}).get("page", 1)) + return _Response(self.pages.get((path, page), {"data": []})) + + +class FullTIMComplianceTests(unittest.TestCase): + def test_per_agent_langfuse_credentials_are_data_driven(self): + from evaluator.config.settings import settings + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + f.write( + "langfuse:\n" + " agents:\n" + " - agentId: arbitrary-new-agent\n" + " host: https://lf-arbitrary.example\n" + " publicKey: pk-arbitrary\n" + " secretKey: sk-arbitrary\n" + ) + path = f.name + old = settings.langfuse_credentials_config_path + try: + settings.langfuse_credentials_config_path = path + c = resolve_langfuse_credentials("arbitrary-new-agent") + self.assertEqual(c.public_key, "pk-arbitrary") + self.assertEqual(c.secret_key, "sk-arbitrary") + self.assertEqual(c.host, "https://lf-arbitrary.example") + self.assertIn("arbitrary-new-agent", c.source) + finally: + settings.langfuse_credentials_config_path = old + Path(path).unlink(missing_ok=True) + + def test_agents_yaml_accepts_document_camel_case(self): + with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f: + f.write("agents:\n - agentId: billing\n daysBack: 2\n percentage: 0.5\n gcs_prefix: agnt_ai_contas\n") + path = f.name + try: + agents = load_agents(path) + self.assertEqual(agents[0].agent_id, "billing") + self.assertEqual(agents[0].days_back, 2) + self.assertEqual(agents[0].percentage, 0.5) + finally: + Path(path).unlink(missing_ok=True) + + def test_pagination_reads_all_pages_without_n_plus_one(self): + c = LangfuseCollector("billing") + # PAGE_SIZE is changed only inside this test to keep fixtures small. + c.PAGE_SIZE = 2 + client = _Client({ + ("/api/public/traces", 1): {"data": [{"id": "t1"}, {"id": "t2"}], "meta": {"totalPages": 2}}, + ("/api/public/traces", 2): {"data": [{"id": "t3"}], "meta": {"totalPages": 2}}, + }) + from evaluator.collectors.langfuse_credentials import LangfuseCredentials + creds = LangfuseCredentials("http://x", "pk", "sk", "test") + rows = asyncio.run(c._fetch_all_pages(client, "/api/public/traces", {}, ("traces",), creds)) + self.assertEqual([x["id"] for x in rows], ["t1", "t2", "t3"]) + self.assertEqual(len(client.calls), 2) + self.assertFalse(any("/api/public/traces/t" in path for path, _ in client.calls)) + + def test_trace_metadata_precedes_session_metadata_and_session_falls_back(self): + c = LangfuseCollector("billing") + trace = { + "id": "t1", "sessionId": "s1", "input": "abc", "output": "ok", + "metadata": {"ura_call_id": "trace-call", "channelId": "trace-channel", "message_id": "trace-message"}, + } + session = {"id": "s1", "metadata": {"ura_call_id": "session-call", "channelId": "session-channel", "message_id": "session-message"}} + record = c._to_record(trace, [], session, "billing") + self.assertEqual(record.metadata["ura_call_id"], "trace-call") + self.assertEqual(record.channel, "trace-channel") + self.assertEqual(record.message_id, "trace-message") + self.assertEqual(record.raw["prompt_length"], 3) + + trace["metadata"] = {} + record2 = c._to_record(trace, [], session, "billing") + self.assertEqual(record2.metadata["ura_call_id"], "session-call") + self.assertEqual(record2.channel, "session-channel") + self.assertEqual(record2.message_id, "session-message") + + def test_export_prompt_length_prefers_raw_trace_count(self): + raw = {"input_text": "normalized", "raw": {"prompt_length": 123}} + self.assertEqual(extract_prompt_length(raw), 123) + + + def test_alias_filter_is_enforced_with_yaml_per_agent_credentials(self): + from evaluator.collectors.langfuse_credentials import LangfuseCredentials + from evaluator.config.settings import settings + + class Collector(LangfuseCollector): + async def _fetch_sessions(self, client, start, end, creds): + return [{"id": "s-agent-a"}, {"id": "s-agent-b"}] + + async def _fetch_traces(self, client, start, end, creds): + return [ + { + "id": "t-agent-a", + "sessionId": "s-agent-a", + "name": "agent-a", + "input": "hello a", + "output": "answer a", + }, + { + "id": "t-agent-b", + "sessionId": "s-agent-b", + "name": "agent-b", + "input": "hello b", + "output": "answer b", + }, + ] + + async def _fetch_observations(self, client, start, end, creds): + return [] + + class NoopAsyncClient: + def __init__(self, *args, **kwargs): + pass + async def __aenter__(self): + return self + async def __aexit__(self, exc_type, exc, tb): + return False + + collector = Collector("agent-a") + creds = LangfuseCredentials( + "https://langfuse-agent-a.example", "pk", "sk", + "/mounted/langfuse_credentials.yaml#agent-a", + ) + old_enable = settings.enable_langfuse + try: + settings.enable_langfuse = True + with patch("evaluator.collectors.langfuse.resolve_langfuse_credentials", return_value=creds), \ + patch("evaluator.collectors.langfuse.httpx.AsyncClient", NoopAsyncClient): + records, sampled, stats = asyncio.run(collector.collect_sampled( + datetime(2026, 8, 13, 0, 0), + datetime(2026, 8, 13, 23, 59, 59), + agent_aliases={"agent-a"}, + percentage=1.0, + seed="agent-a:2026-08-13", + )) + self.assertEqual(sampled, ["s-agent-a"]) + self.assertEqual([r.trace_id for r in records], ["t-agent-a"]) + self.assertEqual(stats["selected_traces"], 1) + self.assertIn("#agent-a", stats["credential_source"]) + finally: + settings.enable_langfuse = old_enable + + def test_evaluator_remains_cronjob_only_and_monitor_is_separate(self): + root = Path(__file__).resolve().parents[1] + k = (root / "deploy/k8s/base/kustomization.yaml").read_text() + cron = (root / "deploy/k8s/base/cronjob.yaml").read_text() + monitor = (root / "deploy/k8s/base/monitor-deployment.yaml").read_text() + self.assertIn("cronjob.yaml", k) + self.assertIn("monitor-deployment.yaml", k) + self.assertIn("monitor-service.yaml", k) + self.assertIn("run-agents", cron) + self.assertNotIn("uvicorn", cron.lower()) + self.assertIn("evaluator_monitor.app:app", monitor) + self.assertIn("suspend: false", cron) + self.assertIn('timeZone: "America/Sao_Paulo"', cron) + + def test_fastapi_is_confined_to_separate_monitor_package(self): + root = Path(__file__).resolve().parents[1] + for path in (root / "evaluator").rglob("*.py"): + source = path.read_text(encoding="utf-8").lower() + self.assertNotIn("import fastapi", source) + self.assertNotIn("import uvicorn", source) + self.assertNotIn("from fastapi", source) + monitor_source = (root / "evaluator_monitor/app.py").read_text(encoding="utf-8").lower() + self.assertIn("from fastapi", monitor_source) + + +if __name__ == "__main__": + unittest.main() + + +def test_metadata_session_recovery_when_langfuse_sessions_endpoint_is_empty(): + from evaluator.collectors.langfuse_credentials import LangfuseCredentials + from evaluator.config.settings import settings + + class Collector(LangfuseCollector): + async def _fetch_sessions(self, client, start, end, creds): + return [] + + async def _fetch_traces(self, client, start, end, creds): + return [ + { + "id": "trace-real", + "sessionId": None, + "name": "agent.gateway_message", + "metadata": {"agent_id": "telecom_contas"}, + "input": "ola", + "output": "resposta", + }, + { + "id": "judge-audit", + "sessionId": None, + "name": "judge-job-telecom_contas", + "metadata": {"agentId": "telecom_contas"}, + }, + ] + + async def _fetch_observations(self, client, start, end, creds): + return [ + { + "id": "obs-1", + "traceId": "trace-real", + "metadata": { + "session_id": "default:telecom_contas:session-123", + "agent_id": "telecom_contas", + "message_id": "msg-1", + }, + } + ] + + class NoopAsyncClient: + def __init__(self, *args, **kwargs): + pass + async def __aenter__(self): + return self + async def __aexit__(self, exc_type, exc, tb): + return False + + collector = Collector("telecom_contas") + creds = LangfuseCredentials("https://lf.example", "pk", "sk", "test#telecom_contas") + old_enable = settings.enable_langfuse + try: + settings.enable_langfuse = True + with patch("evaluator.collectors.langfuse.resolve_langfuse_credentials", return_value=creds), \ + patch("evaluator.collectors.langfuse.httpx.AsyncClient", NoopAsyncClient): + records, sampled, stats = asyncio.run(collector.collect_sampled( + datetime(2026, 8, 14, 0, 0), + datetime(2026, 8, 14, 23, 59, 59), + agent_aliases={"telecom_contas"}, + percentage=1.0, + seed="telecom_contas:2026-08-14", + )) + + assert sampled == ["default:telecom_contas:session-123"] + assert len(records) == 1 + assert records[0].trace_id == "trace-real" + assert records[0].session_id == "default:telecom_contas:session-123" + assert stats["native_sessions_read"] == 0 + assert stats["metadata_sessions_recovered"] == 1 + assert stats["sessions_read"] == 1 + assert stats["traces_without_session"] == 0 + assert stats["selected_traces"] == 1 + finally: + settings.enable_langfuse = old_enable + + +def test_trace_id_is_never_used_as_session_id_fallback(): + from evaluator.collectors.langfuse import _trace_session_id + assert _trace_session_id({"id": "trace-only", "metadata": {}}) is None diff --git a/tests/test_monitor_and_run_id.py b/tests/test_monitor_and_run_id.py new file mode 100644 index 0000000..c0dcff5 --- /dev/null +++ b/tests/test_monitor_and_run_id.py @@ -0,0 +1,25 @@ +from pathlib import Path + + +def test_run_agents_announces_run_id_and_progress_command(): + source = Path("evaluator/cli.py").read_text(encoding="utf-8") + assert "RUN_ID=" in source + assert "python -m evaluator.cli progress" in source + assert "progress_callback=announce_run" in source + + +def test_monitor_is_separate_from_batch_cronjob(): + cron = Path("deploy/k8s/base/cronjob.yaml").read_text(encoding="utf-8") + monitor = Path("deploy/k8s/base/monitor-deployment.yaml").read_text(encoding="utf-8") + service = Path("deploy/k8s/base/monitor-service.yaml").read_text(encoding="utf-8") + assert "run-agents" in cron + assert "uvicorn" not in cron + assert "evaluator_monitor.app:app" in monitor + assert "kind: Service" in service + assert "type: ClusterIP" in service + + +def test_monitor_routes_present(): + source = Path("evaluator_monitor/app.py").read_text(encoding="utf-8") + for route in ("/health", "/api/runs", "/api/runs/{run_id}", "/api/runs/{run_id}/results", "/ui"): + assert route in source diff --git a/tests/test_no_product_hardcodes.py b/tests/test_no_product_hardcodes.py new file mode 100644 index 0000000..f3ee31c --- /dev/null +++ b/tests/test_no_product_hardcodes.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import ast +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from evaluator.publishers.gcs_export import _load_bucket_mapping, resolve_gcs_bucket + + +ROOT = Path(__file__).resolve().parents[1] +OFFICIAL_AGENT_IDS = { + "billing", + "sales", + "collection", + "backoffice-anatel", + "backoffice_anatel", + "telecom_contas", + "retail_orders", + "financeiro_agent", +} + + +class NoProductHardcodesTests(unittest.TestCase): + def test_agent_ids_are_not_python_string_literals_in_runtime(self): + offenders = [] + for path in (ROOT / "evaluator").rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Constant) and isinstance(node.value, str): + if node.value.lower() in OFFICIAL_AGENT_IDS: + offenders.append(f"{path.relative_to(ROOT)}:{node.lineno}: {node.value}") + self.assertEqual(offenders, []) + + def test_storage_buckets_are_loaded_from_configuration(self): + with tempfile.TemporaryDirectory() as td: + cfg = Path(td) / "storage.yaml" + cfg.write_text( + "storage:\n buckets:\n sandbox: bucket-from-config\n", + encoding="utf-8", + ) + with patch("evaluator.publishers.gcs_export.settings.storage_config_path", str(cfg)), \ + patch("evaluator.publishers.gcs_export.settings.judge_gcs_bucket", None), \ + patch("evaluator.publishers.gcs_export.settings.app_env", "sandbox"): + self.assertEqual(_load_bucket_mapping(), {"sandbox": "bucket-from-config"}) + self.assertEqual(resolve_gcs_bucket(), "bucket-from-config") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tim_spec.py b/tests/test_tim_spec.py new file mode 100644 index 0000000..5dac84d --- /dev/null +++ b/tests/test_tim_spec.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import asyncio +import unittest +from datetime import date, datetime +from pathlib import Path + +from evaluator.analytics.vloop import max_consecutive_intent_streak +from evaluator.config.agents import load_agents +from evaluator.core.models import ConversationRecord +from evaluator.engine import EvaluationEngine +from evaluator.judges.llm_judge import TIMStyleLLMJudge +from evaluator.output.legacy_exporter import extract_prompt_length +from evaluator.time_utils import brt_day_window, to_utc_z + + +class FakeLLM: + def __init__(self, replies): + self.replies = list(replies) + self.calls = 0 + + async def complete(self, prompt): + self.calls += 1 + return self.replies.pop(0) + + +class TIMSpecTests(unittest.TestCase): + def test_brt_day_converts_to_expected_utc(self): + start, end = brt_day_window(date(2026, 8, 12)) + self.assertEqual(to_utc_z(start), "2026-08-12T03:00:00Z") + self.assertTrue(to_utc_z(end).startswith("2026-08-13T02:59:59.999999Z")) + + def test_sampling_is_session_based_and_deterministic(self): + records = [ + ConversationRecord(session_id="s1", input_text="a"), + ConversationRecord(session_id="s1", input_text="b"), + ConversationRecord(session_id="s2", input_text="c"), + ConversationRecord(session_id="s3", input_text="d"), + ] + engine = object.__new__(EvaluationEngine) + first, first_sessions = EvaluationEngine._sample_sessions(engine, records, 0.5, "billing", date(2026, 8, 12)) + second, second_sessions = EvaluationEngine._sample_sessions(engine, records, 0.5, "billing", date(2026, 8, 12)) + self.assertEqual(first_sessions, second_sessions) + self.assertEqual({r.session_id for r in first}, set(first_sessions)) + for sid in first_sessions: + self.assertEqual( + sum(1 for r in first if r.session_id == sid), + sum(1 for r in records if r.session_id == sid), + ) + + def test_loop_is_max_consecutive_intent_streak(self): + records = [ + {"metadata": {"intent": "billing"}}, + {"metadata": {"intent": "billing"}}, + {"metadata": {"intent": "billing"}}, + {"metadata": {"intent": "other"}}, + {"metadata": {"intent": "other"}}, + ] + self.assertEqual(max_consecutive_intent_streak(records), 3) + + def test_prompt_length_is_raw_input_character_count(self): + self.assertEqual(extract_prompt_length({"input_text": "12345", "output_text": "ignored"}), 5) + + def test_official_agents_and_prefixes(self): + mapping = {a.agent_id: a.gcs_prefix for a in load_agents()} + self.assertEqual(mapping, { + "billing": "agnt_ai_contas", + "sales": "agnt_ai_ofertas", + "collection": "agnt_ai_cobranca", + "backoffice-anatel": "agnt_ai_bko", + }) + + def test_cronjob_is_not_suspended(self): + text = Path("deploy/k8s/base/cronjob.yaml").read_text() + self.assertIn('timeZone: "America/Sao_Paulo"', text) + self.assertIn("suspend: false", text) + + def test_trace_parse_retry_success_and_failure(self): + async def scenario(): + llm = FakeLLM([ + "not-json", + '{"judgeScore":0.8,"accuracyScore":0.9,"alucinationScore":0.1,"rationale":"ok"}', + ]) + judge = TIMStyleLLMJudge(llm=llm) + result = await judge.judge_trace(ConversationRecord(session_id="s", input_text="x", output_text="y")) + self.assertEqual(llm.calls, 2) + self.assertEqual(result.judgeScore, 0.8) + + llm2 = FakeLLM(["bad", "still-bad"]) + judge2 = TIMStyleLLMJudge(llm=llm2) + result2 = await judge2.judge_trace(ConversationRecord(session_id="s", input_text="x", output_text="y")) + self.assertEqual(llm2.calls, 2) + self.assertIsNone(result2.judgeScore) + self.assertIsNone(result2.accuracyScore) + self.assertIsNone(result2.alucinationScore) + + asyncio.run(scenario()) + + +if __name__ == "__main__": + unittest.main()