From 56a420f5f8a4330d2863141ae7d6dd740df61645 Mon Sep 17 00:00:00 2001 From: Cristiano Hoshikawa Date: Fri, 12 Jun 2026 11:36:33 -0300 Subject: [PATCH] first commit --- .DS_Store | Bin 0 -> 14340 bytes .env.example | 38 + .idea/.gitignore | 12 + .idea/agent_framework_evaluator.iml | 9 + .idea/codeStyles/Project.xml | 7 + .idea/codeStyles/codeStyleConfig.xml | 5 + .idea/misc.xml | 6 + .idea/modules.xml | 8 + .oca/custom_code_review_guidelines.txt | 24 + Dockerfile | 5 + README.en-US.md | 1546 +++++++++++++++++ README.md | 1546 +++++++++++++++++ configs/.DS_Store | Bin 0 -> 6148 bytes configs/identity.yaml | 55 + configs/judge/agents.yaml | 21 + configs/judge/session_metrics.yaml | 17 + configs/judge/trace_metrics.yaml | 16 + configs/llm_profiles/llm_profiles.yaml | 11 + evaluator/.DS_Store | Bin 0 -> 10244 bytes evaluator/__init__.py | 0 .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 206 bytes evaluator/__pycache__/cli.cpython-313.pyc | Bin 0 -> 7804 bytes evaluator/__pycache__/engine.cpython-313.pyc | Bin 0 -> 12636 bytes evaluator/analytics/__init__.py | 0 .../__pycache__/__init__.cpython-313.pyc | Bin 0 -> 216 bytes .../__pycache__/vloop.cpython-313.pyc | Bin 0 -> 4319 bytes evaluator/analytics/vloop.py | 86 + .../api/__pycache__/main.cpython-313.pyc | Bin 0 -> 3466 bytes evaluator/api/main.py | 37 + evaluator/cli.py | 80 + evaluator/collectors/.DS_Store | Bin 0 -> 6148 bytes .../agent_framework.cpython-313.pyc | Bin 0 -> 4879 bytes .../__pycache__/base.cpython-313.pyc | Bin 0 -> 1003 bytes .../__pycache__/langfuse.cpython-313.pyc | Bin 0 -> 19001 bytes .../__pycache__/mock.cpython-313.pyc | Bin 0 -> 1575 bytes evaluator/collectors/agent_framework.py | 42 + evaluator/collectors/base.py | 8 + evaluator/collectors/langfuse.py | 355 ++++ evaluator/collectors/mock.py | 9 + evaluator/config/.DS_Store | Bin 0 -> 6148 bytes .../config/__pycache__/agents.cpython-313.pyc | Bin 0 -> 2542 bytes .../__pycache__/settings.cpython-313.pyc | Bin 0 -> 8648 bytes evaluator/config/agents.py | 36 + evaluator/config/settings.py | 140 ++ .../core/__pycache__/models.cpython-313.pyc | Bin 0 -> 3280 bytes evaluator/core/models.py | 56 + evaluator/engine.py | 144 ++ .../__pycache__/resolver.cpython-313.pyc | Bin 0 -> 2400 bytes evaluator/identity/resolver.py | 41 + .../__pycache__/llm_judge.cpython-313.pyc | Bin 0 -> 6079 bytes evaluator/judges/llm_judge.py | 81 + .../llm/__pycache__/client.cpython-313.pyc | Bin 0 -> 5102 bytes .../profile_resolver.cpython-313.pyc | Bin 0 -> 5699 bytes evaluator/llm/client.py | 98 ++ evaluator/llm/profile_resolver.py | 80 + evaluator/output/.DS_Store | Bin 0 -> 6148 bytes .../legacy_exporter.cpython-313.pyc | Bin 0 -> 13704 bytes evaluator/output/legacy_exporter.py | 278 +++ .../__pycache__/oracle_store.cpython-313.pyc | Bin 0 -> 15600 bytes .../__pycache__/repository.cpython-313.pyc | Bin 0 -> 27628 bytes evaluator/persistence/oracle_store.py | 282 +++ evaluator/persistence/repository.py | 410 +++++ .../__pycache__/loader.cpython-313.pyc | Bin 0 -> 862 bytes evaluator/prompts/loader.py | 11 + .../langfuse_scores.cpython-313.pyc | Bin 0 -> 2541 bytes evaluator/publishers/langfuse_scores.py | 22 + k8s/cronjob.yaml | 20 + output/.DS_Store | Bin 0 -> 6148 bytes pyproject.toml | 25 + 69 files changed, 5667 insertions(+) create mode 100644 .DS_Store create mode 100644 .env.example create mode 100644 .idea/.gitignore create mode 100644 .idea/agent_framework_evaluator.iml create mode 100644 .idea/codeStyles/Project.xml create mode 100644 .idea/codeStyles/codeStyleConfig.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .oca/custom_code_review_guidelines.txt create mode 100644 Dockerfile create mode 100644 README.en-US.md create mode 100644 README.md create mode 100644 configs/.DS_Store create mode 100644 configs/identity.yaml create mode 100644 configs/judge/agents.yaml create mode 100644 configs/judge/session_metrics.yaml create mode 100644 configs/judge/trace_metrics.yaml create mode 100644 configs/llm_profiles/llm_profiles.yaml create mode 100644 evaluator/.DS_Store create mode 100644 evaluator/__init__.py create mode 100644 evaluator/__pycache__/__init__.cpython-313.pyc create mode 100644 evaluator/__pycache__/cli.cpython-313.pyc create mode 100644 evaluator/__pycache__/engine.cpython-313.pyc create mode 100644 evaluator/analytics/__init__.py create mode 100644 evaluator/analytics/__pycache__/__init__.cpython-313.pyc create mode 100644 evaluator/analytics/__pycache__/vloop.cpython-313.pyc create mode 100644 evaluator/analytics/vloop.py create mode 100644 evaluator/api/__pycache__/main.cpython-313.pyc create mode 100644 evaluator/api/main.py create mode 100644 evaluator/cli.py create mode 100644 evaluator/collectors/.DS_Store create mode 100644 evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc create mode 100644 evaluator/collectors/__pycache__/base.cpython-313.pyc create mode 100644 evaluator/collectors/__pycache__/langfuse.cpython-313.pyc create mode 100644 evaluator/collectors/__pycache__/mock.cpython-313.pyc create mode 100644 evaluator/collectors/agent_framework.py create mode 100644 evaluator/collectors/base.py create mode 100644 evaluator/collectors/langfuse.py create mode 100644 evaluator/collectors/mock.py create mode 100644 evaluator/config/.DS_Store create mode 100644 evaluator/config/__pycache__/agents.cpython-313.pyc create mode 100644 evaluator/config/__pycache__/settings.cpython-313.pyc create mode 100644 evaluator/config/agents.py create mode 100644 evaluator/config/settings.py create mode 100644 evaluator/core/__pycache__/models.cpython-313.pyc create mode 100644 evaluator/core/models.py create mode 100644 evaluator/engine.py create mode 100644 evaluator/identity/__pycache__/resolver.cpython-313.pyc create mode 100644 evaluator/identity/resolver.py create mode 100644 evaluator/judges/__pycache__/llm_judge.cpython-313.pyc create mode 100644 evaluator/judges/llm_judge.py create mode 100644 evaluator/llm/__pycache__/client.cpython-313.pyc create mode 100644 evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc create mode 100644 evaluator/llm/client.py create mode 100644 evaluator/llm/profile_resolver.py create mode 100644 evaluator/output/.DS_Store create mode 100644 evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc create mode 100644 evaluator/output/legacy_exporter.py create mode 100644 evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc create mode 100644 evaluator/persistence/__pycache__/repository.cpython-313.pyc create mode 100644 evaluator/persistence/oracle_store.py create mode 100644 evaluator/persistence/repository.py create mode 100644 evaluator/prompts/__pycache__/loader.cpython-313.pyc create mode 100644 evaluator/prompts/loader.py create mode 100644 evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc create mode 100644 evaluator/publishers/langfuse_scores.py create mode 100644 k8s/cronjob.yaml create mode 100644 output/.DS_Store create mode 100644 pyproject.toml diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..c3caf72a70032d21bf7cec7a69babb9a1dbd3528 GIT binary patch literal 14340 zcmeHNX>c4z74A2>X3&f^@)fPtvesT7svzN&>`1Z|!s;qVSFBj~5lbuWL>hawqwMTT ziVvc2I6y*@qJUuXgTllG1xdI`3aG*)6oD8L5H3?-3IieVgE&wSsDe}hue)b=XLeU4 zE+7zRrfRyU-|N@MoA+LS-7|y`Xi0=?2?-EFM7(LP%9xxa#7A@6Mfvp3y7G_$cvE3~ zRtX7_2uWZXCq`E7Y=VF!DxZ(HL^QJCkdH-XYWt>*xp^j-A2{9CHWe|Zuq2*)aK|j8 zx{0d+R|BpFTn)Gya5ZqH8lY=0UScuN-PF~9s{vO7DGkv0;KQ5dK`US8ISU<_PzX=? z6vDF$pK%|+Xg(_sTKO{1nJCIIQOvn0SX&I>;z;+=>X-+ue3|E5oB$UmVD${v4h2~6 zDCc9<2?TlWrmhBD4PYnXamH=-lM@zq$vqoxsyx3YF8R5f902Q_R~U(njHaqMX=?fAev zZN?nbVkYMPV1Q3tdbFghV(X4K2d>-qwzeC4`_HSTYuBycpeUnySc}H>TXiiqu&9UQ zvm+sMT8oDD=*-Z9Zt7DJtv?!?(@Nn+Njkr-bgk+s_iPeXzc`$j(r-)X@!J_ysq}1- z)Gv$UVH`}45vDDMdHqVyc2OLo^wyY$DJLsco@*sFB=(!A6W&CPvP)7;ssD%`wM6Tb zJ)#sOxP%RDPsmUD9$K>Q@PphbYM?6IP?hcvp z1M^xmJ*xepI6^ti;V=@_j?S*`9*<`=EsUD8p_q1HF{;JX6;WjMZ^ruU-6My#-yYWE zM$BRkFs7Ta`p?eu=*@I3sdP=KBvoV|X(5AnHe{0H zd@pLkvtvzychDcfq^iE_e^z3lG3Y;G^&$d;%VZN8oetdH5VSK1Xib?ZH|(S>+kJ$&g~OSc zTQK{`v}aGbcT>e?#W#R{qb;pwz-kfo5b->%z16Sy)B^M@UffxQ#X|HdUaYS1`BVX& znHOv8e2SWft|~Fv`Wh^PRoK_0C~7`Bt#p1m&u<2(B#)a85(*uTnA0ij0WBZ127CDFp5U5!3@kYLyx1GFTxVs3GYNxKZ(YE zKYRc_2p@tE!^hy`aGDwXqs-vH0AGWz!#Ch8JOSt6`|vFM0DcH7a1nkDm*I~Y27Qy$ zpy5xs4SLQi_H`B~f1aKTwRkQ}bG&=-qFjOYwsaY)fKkNlGT5?cM~O|>%idH#<{^@j zgUw!VMxdo}jE$-tN0o*SNTj&P+v)8V5gOQlLXzGqirFwB$?T`hESMmruz?C`ypV(m z4rQf49*dxLq>XfwNfIM>k^9MM@(B3~f`%u^lVpWlB)=gqkiQ^o$ipaF1R|7LpkXuY zggsCX4bX@YwFhp30~k?nfe8j34ub)=G3am{-ob#wNw^PT!zl(E9zV`o+ zxnCfc$nyvp{)*;ajQDs1Y=asG8QKvtbfbay!a*2=aX5r#9)iPg1m++L^Kd&Hh2wAn z?m%?>Ubq|Xu`u!}#K;f98Tcf83O)^AgfGEk@VE^te;3XFNANRv{uKek%b6x^Ux5Gd z{!3@JbEx20RP4*}YIU58v5%_M(MAR^D8NNp!`($Z6C6-MV&mG!hn)K?fZ(X%00}v2 zILF3%_+KU->^z^mh0GA0#GMQJ9dEbu>~p8z->dk$Ny*yZOD?`+-Y`k$T!8)RB#gQb zIeT80aB}T^f2OD4p7p$Mo4mdE4O<^LvnH#CqkqI?c+s&NWID6=y#02by>CL6H$Tfk zVM^BCb1uHKU$(z3!afZWCAZ@2XNY-~K*Nk#Ip63`x>?Aqk%ouv8Iya)lsaj%K}YwD z>EAzN%J#Y>Y;@0;-1DW3Bh~-+^Q8=9wlB^r)m0lUV31WS4a!!p0BpWR|AA__y6?ey8HkC@&^KU|L^Yq zGXlA7|L)wiP=~yV_x}ZKek4Hu|7&0U|9@V~O + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..919ce1f --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..89ee753 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..589bacc --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.oca/custom_code_review_guidelines.txt b/.oca/custom_code_review_guidelines.txt new file mode 100644 index 0000000..a0a3b63 --- /dev/null +++ b/.oca/custom_code_review_guidelines.txt @@ -0,0 +1,24 @@ +# Sample guideline, please follow similar structure for guideline with code samples +# 1. Suggest using streams instead of simple loops for better readability. +# +# *Comment: +# Category: Minor +# Issue: Use streams instead of a loop for better readability. +# Code Block: +# +# ```java +# // Calculate squares of numbers +# List squares = new ArrayList<>(); +# for (int number : numbers) { +# squares.add(number * number); +# } +# ``` +# Recommendation: +# +# ```java +# // Calculate squares of numbers +# List squares = Arrays.stream(numbers) +# .map(n -> n * n) // Map each number to its square +# .toList(); +# ``` +# diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0c7addf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.12-slim +WORKDIR /app +COPY . /app +RUN pip install --no-cache-dir -e . +CMD ["python", "-m", "evaluator.cli", "run-agents", "--source", "langfuse"] diff --git a/README.en-US.md b/README.en-US.md new file mode 100644 index 0000000..97ea9db --- /dev/null +++ b/README.en-US.md @@ -0,0 +1,1546 @@ +# agent_framework_evaluator + +## 1. What is the `agent_framework_evaluator`? + +The `agent_framework_evaluator` is a complementary service to the `agent_framework_oci` created to evaluate real conversations conducted by the framework's agents. + +It collects conversations from a source, usually Langfuse, reconstructs the context of the interaction, runs a Judge LLM, writes the results to an Oracle/ADB database, generates legacy files in TXT.GZ format, and optionally publishes scores back to Langfuse. + +In simple terms: + +```text +agent_framework_oci gera conversas e telemetria + ↓ +Langfuse armazena traces, spans, generations, metadata e usage + ↓ +agent_framework_evaluator coleta essas conversas + ↓ +LLM Judge avalia qualidade, precisão, alucinação, resolução e CSI + ↓ +Oracle/ADB persiste runs, itens, resultados, achados e progresso + ↓ +Exporter gera arquivo legado AGENTE__LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +The evaluator does not replace the guardrails, online judges, or telemetry of `agent_framework_oci`. It acts as an offline/batch layer for evaluation, auditing, and export. + +--- + +## 2. Purpose of the solution + +The purpose of the evaluator is to allow conversations that have already taken place to be analyzed later using standardized criteria. + +It mainly serves these scenarios: + +- daily evaluation of conversations by agent; +- generation of legacy evaluation files; +- auditing the quality of responses; +- identification of hallucination, low accuracy, low resolution or poor customer experience; +- comparison between agents such as `telecom_contas`, `retail_orders` and `financeiro_agent`; +- optional publication of scores on Langfuse; +- persistence of evaluation history in Oracle/ADB; +- progress tracking via API or CLI. + +--- + +## 3. How it integrates with `agent_framework_oci` + +`agent_framework_oci` is the main runtime for agents. It executes the conversational flow with LangGraph, supervisor, guardrails, judges, MCP tools, memory, RAG, and telemetry. + +During execution, the framework publishes traces to Langfuse containing: + +- `trace_id`; +- `session_id`; +- `message_id`; +- `agent_id`; +- `channel`; +- canonical `business_context`; +- IC/NOC/GRL events; +- LangGraph spans; +- guardrail spans; +- judge spans; +- LLM generations; +- model usage, when available; +- `prompt_tokens`, `completion_tokens` and `total_tokens`, when returned by the provider; +- `input_size`, when emitted by the framework spans. + +The evaluator uses this telemetry as a data source. + +The main integration happens like this: + +```text +agent_framework_oci +├── Executes agents +├── Resolves identity via identity.yaml +├── Creates canonical BusinessContext +├── Calls MCP/RAG/LLM +├── Emits Langfuse telemetry +└── Writes usage/model/tokens when available + +agent_framework_evaluator +├── Reads traces in Langfuse +├── Applies identity.yaml to normalize identity +├── Rebuilds ConversationRecord +├── Executes LLM Judge offline +├── Writes results to Oracle/ADB +├── Exports legacy TXT.GZ +└── Optionally publish scores on Langfuse +``` + +--- + +## 4. General architecture + +```text ++------------------------+ +| agent_framework_oci | +|------------------------| +| LangGraph | +| Supervisor | +| Guardrails | +| Judges online | +| MCP Tool Router | +| RAG | +| Memory / Checkpoint | +| Langfuse Telemetry | ++-----------+------------+ + | + v ++------------------------+ +| Langfuse | +|------------------------| +| Traces | +| Spans | +| Generations | +| Metadata | +| Usage / Tokens | ++-----------+------------+ + | + v ++------------------------+ +| agent_framework_ | +| evaluator | +|------------------------| +| Collectors | +| Identity Resolver | +| Conversation Records | +| LLM Judge | +| VLoop analytics | +| Repository Oracle | +| Legacy Exporter | +| API / CLI | ++-----------+------------+ + | + v ++------------------------+ +| Oracle ADB | +|------------------------| +| EVALUATION_RUN | +| EVALUATION_ITEM | +| EVALUATION_RESULT | +| EVALUATION_FINDING | +| EVALUATION_PROGRESS | +| EVALUATION_METRIC | ++-----------+------------+ + | + v ++------------------------+ +| Output | +|------------------------| +| TXT.GZ legacy | +| API dashboard | +| Langfuse scores | ++------------------------+ +``` + +--- + +## 5. Solution components + +### 5.1 CLI + +Main file: + +```text +evaluator/cli.py +``` + +Responsible for exposing commands such as: + +```bash +python -m evaluator.cli init-db +python -m evaluator.cli show-config +python -m evaluator.cli run --source langfuse +python -m evaluator.cli run-agents --source langfuse +python -m evaluator.cli runs +python -m evaluator.cli progress +``` + +The CLI is the main way to operate the evaluator in batch mode. + +--- + +### 5.2 API + +Main file: + +```text +evaluator/api/main.py +``` + +Exposes HTTP endpoints to query progress, runs, and results. + +Expected examples: + +```text +GET /health +GET /runs +GET /runs/{run_id}/progress +GET /runs/{run_id}/results +GET /runs/{run_id}/findings +``` + +The API allows you to build a simple graphical interface or integrate the evaluator with other systems. + +--- + +### 5.3 EvaluationEngine + +Main file: + +```text +evaluator/engine.py +``` + +It is the central orchestrator of the evaluator. + +Responsibilities: + +1. create a new evaluation run (`EVALUATION_RUN`); +2. choose the collector according to the `source`; +3. collect conversations; +4. apply sampling by agent; +5. insert items into `EVALUATION_ITEM`; +6. process each item; +7. call the LLM Judge; +8. save trace result; +9. run session evaluation; +10. save session result; +11. export legacy file; +12. mark final execution status; +13. issue progress events. + +Simplified flow: + +```text +run_agent() + ↓ +collector.collect() + ↓ +repository.insert_items() + ↓ +_process() + ↓ +judge.judge_trace() + ↓ +repository.save_trace_result() + ↓ +judge.judge_sessions() + ↓ +repository.save_session_result() + ↓ +export_legacy_txt_gz() +``` + +--- + +### 5.4 Collectors + +Directory: + +```text +evaluator/collectors/ +``` + +Collectors are responsible for fetching conversations from an external source and converting them to `ConversationRecord`. + +Typical collectors: + +```text +evaluator/collectors/langfuse.py +evaluator/collectors/agent_framework.py +evaluator/collectors/mock.py +evaluator/collectors/base.py +``` + +#### LangfuseCollector + +This is the main collector. + +Responsibilities: + +- search for traces in Langfuse; +- filter by period; +- filter by agent/alias; +- retrieve trace details; +- extract input/output; +- reconstruct messages; +- collect metadata; +- apply `identity.yaml`; +- assemble canonical `BusinessContext`; +- fill in `ConversationRecord`. + +The collector must normalize data so that the exporter does not need to know Langfuse's internal details. + +--- + +### 5.5 Identity Resolver + +Recommended directory: + +```text +evaluator/identity/ +``` + +Main file: + +```text +evaluator/identity/resolver.py +``` + +The evaluator must use the same identity concept as `agent_framework_oci`, based on the file: + +```text +configs/identity.yaml +``` + +The function of `identity.yaml` is to map variable input fields to a canonical model: + +```text +customer_key +contract_key +interaction_key +account_key +resource_key +session_key +``` + +Conceptual example: + +```yaml +identity: + version: 2 + keys: + customer_key: + sources: + - business_context.customer_key + - metadata.customer_key + - user_id + contract_key: + sources: + - business_context.contract_key + - metadata.contract_key + interaction_key: + sources: + - business_context.interaction_key + - metadata.ura_call_id + - metadata.message_id + - message_id + session_key: + sources: + - business_context.session_key + - session_id + - conversation_key +``` + +With this, the evaluator is not directly tied to fields such as `ura_call_id`, `call_id`, `message_id` or `interaction_key`. It resolves everything to `interaction_key`. + +--- + +### 5.6 Models + +Main file: + +```text +evaluator/core/models.py +``` + +Defines the core objects of the evaluator. + +Main models: + +```python +class ConversationRecord +class ConversationMessage +class TraceJudgeResult +class SessionJudgeResult +class CombinedJudgeResult +class RunStatus +class ItemStatus +``` + +#### ConversationRecord + +Represents an evaluated conversation or turn. + +Common fields: + +```text +trace_id +session_id +message_id +agent_id +channel +input_text +output_text +messages +metadata +raw +``` + +The `metadata` field must contain normalized data: + +```text +business_context +uraCallId +channelId +messageId +promptLength +``` + +The `raw` field keeps the original payload for auditing and fallback. + +--- + +### 5.7 LLM Judge + +Main file: + +```text +evaluator/judges/llm_judge.py +``` + +Main class: + +```python +TIMStyleLLMJudge +``` + +Responsibilities: + +- load evaluation prompts; +- set up trace prompt; +- set up session prompt; +- call LLM via configured client; +- interpret JSON response; +- return `TraceJudgeResult` and `SessionJudgeResult`. + +The judge evaluates metrics such as: + +```text +judgeScore +accuracyScore +alucinationScore +inferredCsiScore +resolution +conversationPrecision +rationale +``` + +The judge must be LLM-based, not deterministic. + +--- + +### 5.8 Prompts + +Directory: + +```text +evaluator/prompts/ +``` + +Expected files: + +```text +trace_judge_prompt.md +session_judge_prompt.md +loader.py +``` + +The trace prompt evaluates an individual response. + +The session prompt evaluates the conversation grouped by `session_id`. + +Example of expected LLM output for trace: + +```json +{ + "judgeScore": 0.8, + "accuracyScore": 0.9, + "alucinationScore": 0.1, + "rationale": "A response that is relevant to the context and based on available data." +} +``` + +Example of expected output for session: + +```json +{ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "The conversation was resolved with consistent information." +} +``` + +--- + +### 5.9 LLM Client + +Directory: + +```text +evaluator/llm/ +``` + +Typical files: + +```text +evaluator/llm/client.py +evaluator/llm/oci_openai.py +``` + +The evaluator must use the same LLM access pattern as `agent_framework_oci`, preferably via the `oci_openai` provider. + +Common variables: + +```env +LLM_PROVIDER=oci_openai +OCI_GENAI_ENDPOINT=... +OCI_GENAI_MODEL_ID=... +OCI_GENAI_API_KEY=... +OCI_GENAI_COMPARTMENT_ID=... +``` + +The client needs to return raw text for the Judge to interpret as JSON. + +--- + +### 5.10 Repository / Oracle Store + +Directory: + +```text +evaluator/persistence/ +``` + +Main files: + +```text +evaluator/persistence/oracle_store.py +evaluator/persistence/repository.py +``` + +`OracleStore` takes care of: + +- connection with ADB/Oracle; +- wallet; +- DSN; +- schema creation/adjustment; +- thread-safe execution for asynchronous calls; +- table prefix. + +The `EvaluationRepository` takes care of: + +- creating runs; +- recording progress; +- inserting items; +- search for next items; +- marking an item as `PROCESSING`, `COMPLETED` or `FAILED`; +- save results; +- save findings; +- summarize run; +- list runs; +- check progress. + +--- + +### 5.11 Legacy Exporter + +Main file: + +```text +evaluator/output/legacy_exporter.py +``` + +Generates the legacy file: + +```text +output/AGENTE__LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +Column format: + +```text +judgeScore +accuracyScore +alucinationScore +promptLength +loop +inferredCsiScore +resolution +conversationPrecision +uraCallId +channelId +sessionId +messageId +``` + +Example: + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"TOTAL"|;"19" +``` + +#### promptLength + +The `promptLength` field must use this priority: + +1. `prompt_tokens`/ `promptTokens` /`input_tokens`/ `inputTokens` in Langfuse observations; +2. `usage.input` or `usageDetails.input`; +3. `metadata.input_size` issued by the framework; +4. fallback for text size of `input_text`, `output_text`, and `messages`. + +Example: + +```text +promptLength = 732 +``` + +#### loop + +The `loop field` uses the VLoop detector. + +```text +0 = sem loop detectado +1 = loop detectado +``` + +--- + +### 5.12 VLoop Analytics + +Main file: + +```text +evaluator/analytics/vloop.py +``` + +Responsible for detecting conversational repetition/loop in a pattern similar to the VLoop guardrail of `agent_framework_oci`. + +The function normally exposed is: + +```python +vloop_flag(raw) -> int +``` + +It returns: + +```text +0 when there is no evidence of a loop +1 when there is suspected repetition +``` + +--- + +### 5.13 Langfuse Score Publisher + +Main file: + +```text +evaluator/publishers/langfuse_scores.py +``` + +Responsible for publishing evaluation scores back to Langfuse, when enabled. + +Control variable: + +```env +PUBLISH_LANGFUSE_SCORES=true +``` + +When disabled, the evaluator only writes to the database and exports the file. + +--- + +## 6. Directory structure + +```text +agent_framework_evaluator/ +├── configs/ +│ ├── identity.yaml +│ └── judge/ +│ └── agents.yaml +├── docs/ +├── evaluator/ +│ ├── __init__.py +│ ├── cli.py +│ ├── engine.py +│ ├── api/ +│ │ └── main.py +│ ├── analytics/ +│ │ └── vloop.py +│ ├── collectors/ +│ │ ├── base.py +│ │ ├── langfuse.py +│ │ ├── agent_framework.py +│ │ └── mock.py +│ ├── config/ +│ │ ├── settings.py +│ │ └── agents.py +│ ├── core/ +│ │ └── models.py +│ ├── identity/ +│ │ └── resolver.py +│ ├── judges/ +│ │ └── llm_judge.py +│ ├── llm/ +│ │ ├── client.py +│ │ └── oci_openai.py +│ ├── output/ +│ │ └── legacy_exporter.py +│ ├── persistence/ +│ │ ├── oracle_store.py +│ │ └── repository.py +│ ├── prompts/ +│ │ ├── loader.py +│ │ ├── trace_judge_prompt.md +│ │ └── session_judge_prompt.md +│ └── publishers/ +│ └── langfuse_scores.py +├── output/ +├── Dockerfile +├── docker-compose.yml +├── pyproject.toml +└── README.md +``` + +--- + +## 7. Configuration + +### 7.1 `.env file` + +Example: + +```env +# Oracle / ADB +ADB_USER=ADMIN +ADB_PASSWORD=your_password +ADB_DSN=oradb23ai_high +ADB_WALLET_DIR=/path/to/Wallet_ORADB23ai +DB_TABLE_PREFIX=AGENTFW_ + +# Langfuse +LANGFUSE_ENABLED=true +LANGFUSE_HOST=http://localhost:3005 +LANGFUSE_PUBLIC_KEY=pk-lf-... +LANGFUSE_SECRET_KEY=sk-lf-... +PUBLISH_LANGFUSE_SCORES=false + +# LLM +LLM_PROVIDER=oci_openai +OCI_GENAI_ENDPOINT=https://... +OCI_GENAI_MODEL_ID=... +OCI_GENAI_API_KEY=... +OCI_GENAI_COMPARTMENT_ID=... + +# Evaluator +EVALUATOR_OUTPUT_DIR=output +EVALUATOR_BATCH_SIZE=10 +EVALUATOR_MAX_ATTEMPTS=2 +EVALUATOR_AGENTS_CONFIG=configs/judge/agents.yaml +IDENTITY_CONFIG_PATH=configs/identity.yaml +TRACE_PROMPT_PATH=evaluator/prompts/trace_judge_prompt.md +SESSION_PROMPT_PATH=evaluator/prompts/session_judge_prompt.md +``` + +--- + +### 7.2 Agent configuration + +File: + +```text +configs/judge/agents.yaml +``` + +Example: + +```yaml +agents: + - agent_id: telecom_contas + enabled: true + aliases: + - telecom_contas + - billing_agent + - financeiro_agent + percentage: 1.0 + + - agent_id: retail_orders + enabled: true + aliases: + - retail_orders + - orders_agent + percentage: 1.0 + + - agent_id: financeiro_agent + enabled: true + aliases: + - financeiro_agent + percentage: 1.0 +``` + +The `aliases` field is important because Langfuse can register the agent in different ways, for example: + +```text +agent_id = telecom_contas +route = financeiro_agent +agent = financeiro_agent +``` + +--- + +### 7.3 Identity configuration + +File: + +```text +configs/identity.yaml +``` + +The evaluator must use the same pattern as the framework. + +Example: + +```yaml +identity: + version: 2 + keys: + customer_key: + sources: + - business_context.customer_key + - metadata.customer_key + - user_id + + contract_key: + sources: + - business_context.contract_key + - metadata.contract_key + + interaction_key: + sources: + - business_context.interaction_key + - metadata.ura_call_id + - metadata.message_id + - message_id + + session_key: + sources: + - business_context.session_key + - metadata.session_key + - session_id + - conversation_key +``` + +The `interaction_key` field is used to populate the `uraCallId` in the legacy export. + +--- + +## 8. How to run + +### 8.1 Install dependencies + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +If you are using Conda: + +```bash +conda activate py313 +pip install -e . +``` + +--- + +### 8.2 Validate configuration + +```bash +python -m evaluator.cli show-config +``` + +Expected output: + +```text +{ + "env_path": ".../.env", + "adb_dsn": "oradb23ai_high", + "wallet": ".../Wallet_ORADB23ai", + "langfuse": true, + "publish_langfuse_scores": false, + "llm_provider": "oci_openai", + "agents_config": "configs/judge/agents.yaml" +} +``` + +--- + +### 8.3 Create/validate schema + +```bash +python -m evaluator.cli init-db +``` + +Expected output: + +```text +{'status': 'OK', 'message': 'Evaluator schema checked/created successfully.'} +``` + +--- + +### 8.4 Run evaluation by period + +```bash +python -m evaluator.cli run \ + --period-start 2026-06-11T00:00:00 \ + --period-end 2026-06-12T00:00:00 \ + --source langfuse +``` + +--- + +### 8.5 Run evaluation by configured agents + +```bash +python -m evaluator.cli run-agents --source langfuse +``` + +Expected output: + +```text +[ + { + 'status': 'COMPLETED', + 'run_id': '...', + 'total_items': 19, + 'completed_items': 19, + 'failed_items': 0, + 'evaluations': 19, + 'avg_score': 0.72, + 'agent_id': 'telecom_contas', + 'output_file': 'output/AGENTE_telecom_contas_LLM_JUDGE_20260612.TXT.GZ', + 'uploaded_to': None + } +] +``` + +--- + +### 8.6 Check progress + +```bash +python -m evaluator.cli progress +``` + +Or via API: + +```bash +curl http://localhost:8001/runs//progress +``` + +--- + +### 8.7 View exported file + +```bash +gzip -cd output/AGENTE_telecom_contas_LLM_JUDGE_20260612.TXT.GZ +``` + +Example of a valid line: + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"TOTAL"|;"19" +``` + +--- + +## 9. Database + +### 9.1 Main tables + +#### EVALUATION_RUN + +Stores an evaluation run. + +Main fields: + +```text +RUN_ID +PERIOD_START +PERIOD_END +SOURCE +AGENT_ID +STATUS +TOTAL_ITEMS +PROCESSED_ITEMS +FAILED_ITEMS +LAST_HEARTBEAT_AT +CREATED_AT +UPDATED_AT +ERROR_MESSAGE +``` + +--- + +#### EVALUATION_ITEM + +Stores each conversation/turn collected. + +Main fields: + +```text +ITEM_ID +RUN_ID +TRACE_ID +SESSION_ID +MESSAGE_ID +AGENT_ID +CHANNEL +STATUS +ATTEMPT_COUNT +RAW_JSON +CREATED_AT +UPDATED_AT +ERROR_MESSAGE +``` + +--- + +#### EVALUATION_RESULT + +Stores trace and session results. + +Main fields: + +```text +RESULT_ID +RUN_ID +ITEM_ID +TRACE_ID +SESSION_ID +AGENT_ID +JUDGE_TYPE +JUDGE_NAME +JUDGE_SCORE +ACCURACY_SCORE +ALUCINATION_SCORE +INFERRED_CSI_SCORE +RESOLUTION +CONVERSATION_PRECISION +RATIONALE +RESULT_JSON +CREATED_AT +``` + +`JUDGE_TYPE` can be: + +```text +TRACE +SESSION +``` + +--- + +#### EVALUATION_PROGRESS_EVENT + +Stores execution progress events. + +Stage examples: + +```text +RUN_CREATED +COLLECTING +COLLECTED +SAMPLED +ITEMS_INSERTED +BATCH_STARTED +ITEM_COMPLETED +ITEM_FAILED +SESSION_JUDGE_COMPLETED +EXPORTED +COMPLETED +PARTIAL +``` + +--- + +## 10. How the codes work together + +### 10.1 Complete execution flow + +```text +CLI run-agents + ↓ +load configs/judge/agents.yaml + ↓ +for each enabled agent + ↓ +EvaluationEngine.run_agent(agent) + ↓ +cria EVALUATION_RUN + ↓ +LangfuseCollector.collect(...) + ↓ +IdentityResolver.resolve(...) + ↓ +ConversationRecord + ↓ +EvaluationRepository.insert_items(...) + ↓ +EvaluationEngine._process(run_id) + ↓ +TIMStyleLLMJudge.judge_trace(record) + ↓ +LLMClient.complete(prompt) + ↓ +save_trace_result(...) + ↓ +TIMStyleLLMJudge.judge_sessions(records) + ↓ +save_session_result(...) + ↓ +export_legacy_txt_gz(...) + ↓ +COMPLETED +``` + +--- + +### 10.2 Role of the collector + +The collector is responsible for transforming external data into canonical data. + +It must hide differences between sources such as: + +```text +Langfuse +agent_framework database +mock data +``` + +The output must always be: + +```python +ConversationRecord +``` + +--- + +### 10.3 Role of the judge + +The judge receives a `ConversationRecord`, assembles a prompt, and calls the LLM. + +It should not know about Oracle, Langfuse, legacy export, or API. + +It only evaluates. + +--- + +### 10.4 Role of the repository + +The repository is the persistence layer. + +It must not contain an evaluation business rule. + +It only writes, retrieves, and updates data. + +--- + +### 10.5 Role of the exporter + +The exporter transforms persisted data into a legacy file. + +It should not resolve identity in a complex way. + +Ideally, it should read fields that are already normalized: + +```text +metadata.business_context.interaction_key +metadata.channelId +metadata.messageId +metadata.promptLength +``` + +However, for resilience, it can also query `RAW_JSON` as a fallback. + +--- + +## 11. Important design rules + +### 11.1 The evaluator must not be anchored to an agent + +Avoid logic like: + +```python +if agent_id == "telecom_contas": + ura_call_id = metadata["ura_call_id"] +``` + +The correct thing to do is to use `identity.yaml`. + +--- + +### 11.2 The exporter must not know internal details of Langfuse + +Avoid excessive coupling to paths such as: + +```text +raw.detail.observations[0].metadata.ura_call_id +raw.trace.input.business_context.interaction_key +``` + +This should be resolved in the collector. + +--- + +### 11.3 `promptLength` should come from tokens when possible + +Recommended priority: + +```text +1. prompt_tokens / promptTokens +2. input_tokens / inputTokens +3. usage.input / usageDetails.input +4. metadata.input_size +5. tamanho textual de input/output/messages +``` + +--- + +### 11.4 `uraCallId` must come from BusinessContext + +The legacy field `uraCallId` must be mapped to: + +```text +business_context.interaction_key +``` + +This is the canonical name of the framework. + +--- + +### 11.5 `sessionId` must come from BusinessContext + +The legacy `sessionId` field must be mapped to: + +```text +business_context.session_key +``` + +Not to be confused with the full composite key: + +```text +default:telecom_contas: +``` + +The evaluator can store the full key, but the legacy export should normally use the clean session identifier. + +--- + +## 12. Recommended tests + +### 12.1 Configuration test + +```bash +python -m evaluator.cli show-config +``` + +Validate: + +```text +ADB_DSN +Wallet +Langfuse enabled +LLM provider +Agents config +Identity config +``` + +--- + +### 12.2 Database test + +```bash +python -m evaluator.cli init-db +``` + +Then validate tables: + +```sql +select table_name +from user_tables +where table_name like 'AGENTFW_EVALUATION%'; +``` + +--- + +### 12.3 Mock test + +```bash +python -m evaluator.cli run --source mock +``` + +Use this test to validate schema, judge, and export without relying on Langfuse. + +--- + +### 12.4 Test with Langfuse + +```bash +python -m evaluator.cli run-agents --source langfuse +``` + +Validate: + +```text +total_items > 0 +completed_items > 0 +failed_items = 0 +evaluations > 0 +output_file preenchido +``` + +--- + +### 12.5 Export test + +```bash +gzip -cd output/AGENTE_telecom_contas_LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +Validate columns: + +```text +judgeScore filled in +accuracyScore filled in +hallucinationScore filled in +promptLength greater than 0 +loop 0 or 1 +inferredCsiScore filled in +resolution 0 or 1 +conversationPrecision 0 or 1 +uraCallId filled in +channelId filled in +sessionId filled in +messageId filled in +``` + +--- + +## 13. Troubleshooting + +### 13.1 `promptLength` outputs 0 + +Common causes: + +- `find_prompt_tokens` was not included in the file; +- `promptTokens` is zeroed in Langfuse; +- `input_size` is not being traversed; +- `RAW_JSON` is coming as an unconverted string; +- old exporter is still running; +- `except Exception: pass` is masking error. + +Recommended debug: + +```python +print("PROMPT_LENGTH", extract_prompt_length(raw)) +print("RAW_TYPE", type(raw)) +print("RAW_KEYS", list(raw.keys())[:20]) +``` + +--- + +### 13.2 `uraCallId` comes out empty + +Common causes: + +- `identity.yaml` is not being loaded; +- collector is not copying `business_context` to `metadata`; +- `interaction_key` does not exist in the trace; +- exporter does not use `business_context.interaction_key`. + +Validation: + +```sql +select RAW_JSON +from AGENTFW_EVALUATION_ITEM +where MESSAGE_ID = ''; +``` + +Search: + +```text +interaction_key +ura_call_id +business_context +``` + +--- + +### 13.3 `ORA-00904 invalid identifier` + +Usually indicates an old schema. + +Examples already found: + +```text +ORA-00904: UPDATED_AT invalid identifier +ORA-00904: REASONING invalid identifier +ORA-00904: JUDGE_TYPE invalid identifier +``` + +Correction: + +```bash +python -m evaluator.cli init-db +``` + +If the table already exists without the new column,`_init_schema` needs to run `ALTER TABLE ADD` in an idempotent manner. + +--- + +### 13.4 `ORA-00054 resource busy` + +Indicates a lock on the table. + +Common causes: + +- API running while `init-db` tries to change schema; +- another process using the table; +- transaction open in SQL Developer. + +Correction: + +1. stop API/CLI; +2. close open sessions; +3. run `init-db` again. + +--- + +### 13.5 `OCI LLM 401` + +Indicates an authentication problem in the LLM. + +Validate: + +```env +OCI_GENAI_ENDPOINT +OCI_GENAI_MODEL_ID +OCI_GENAI_API_KEY +OCI_GENAI_COMPARTMENT_ID +``` + +Also confirm that the evaluator is reading the correct `.env`: + +```bash +python -m evaluator.cli show-config +``` + +--- + +### 13.6 `Entity with key ${OCI_GENAI_MODEL_ID} not found` + +Indicates that the literal value `${OCI_GENAI_MODEL_ID}` has reached the provider. + +Common causes: + +- variable not expanded; +- YAML using `${OCI_GENAI_MODEL_ID}` without interpolation; +- `.env` not loaded; +- LLM client configuration does not resolve placeholders. + +Correction: + +- put the real model ID in `the .env`; +- ensure interpolation in `settings.py`; +- validate with `show-config`. + +--- + +## 14. Final validation checklist + +Before considering the evaluator ready, validate: + +```text +[ ] init-db executes without error +[ ] show-config displays correct .env file +[ ] Langfuse returns traces +[ ] run-agents collects items per agent +[ ] LLM Judge responds with valid JSON +[ ] EVALUATION_RESULT records TRACE and SESSION data +[ ] progress displays useful events +[ ] export TXT.GZ is generated +[ ] promptLength > 0 +[ ] uraCallId populated +[ ] sessionId populated +[ ] messageId populated +[ ] loop populated with 0 or 1 +[ ] file ends with TOTAL +[ ] scores can be published to Langfuse when enabled +``` + +--- + +## 15. Example of validated final result + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"0.9"|;"1"|;"0"|;"642"|;"0"|;"0.5"|;"1"|;"1"|;"5ab3ea80-7428-402f-98ec-04e7cd5327e4"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"5ab3ea80-7428-402f-98ec-04e7cd5327e4" +"TOTAL"|;"19" +``` + +This result indicates: + +- Judge working; +- prompt tokens extracted correctly; +- VLoop without occurrence; +- session metrics filled in; +- canonical identity working; +- legacy export in the expected layout. + +--- + +## 16. Executive summary + +The `agent_framework_evaluator` is the batch/offline evaluation layer of the `agent_framework_oci` ecosystem. + +It consumes the telemetry generated by the framework, applies a Judge LLM with evaluation rules, persists results in Oracle/ADB, generates a file, and can republish scores in Langfuse. + +The correct architecture separates responsibilities: + +```text +Collector normalizes data. +IdentityResolver resolves identity. +Judge evaluates conversation. +Repository persists data. +Exporter generates legacy data. +API/CLI operate the solution. +``` + +This makes the evaluator generic for multiple agents and avoids direct coupling to specific trace or payload formats. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b61a9dc --- /dev/null +++ b/README.md @@ -0,0 +1,1546 @@ +# agent_framework_evaluator + +## 1. O que é o `agent_framework_evaluator`? + +O `agent_framework_evaluator` é um serviço complementar ao `agent_framework_oci` criado para avaliar conversas reais executadas pelos agentes do framework. + +Ele coleta conversas de uma fonte, normalmente o Langfuse, reconstrói o contexto da interação, executa um Judge LLM, grava os resultados em banco Oracle/ADB, gera arquivos legados no formato TXT.GZ e, opcionalmente, publica scores de volta no Langfuse. + +Em termos simples: + +```text +agent_framework_oci gera conversas e telemetria + ↓ +Langfuse armazena traces, spans, generations, metadata e usage + ↓ +agent_framework_evaluator coleta essas conversas + ↓ +LLM Judge avalia qualidade, precisão, alucinação, resolução e CSI + ↓ +Oracle/ADB persiste runs, itens, resultados, achados e progresso + ↓ +Exporter gera arquivo legado AGENTE__LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +O evaluator não substitui os guardrails, judges online ou telemetria do `agent_framework_oci`. Ele atua como uma camada offline/batch de avaliação, auditoria e exportação. + +--- + +## 2. Objetivo da solução + +O objetivo do evaluator é permitir que conversas já executadas sejam analisadas posteriormente com critérios padronizados. + +Ele atende principalmente estes cenários: + +- avaliação diária de conversas por agente; +- geração de arquivos legados de avaliação; +- auditoria de qualidade de respostas; +- identificação de alucinação, baixa precisão, baixa resolução ou baixa experiência do cliente; +- comparação entre agentes como `telecom_contas`, `retail_orders` e `financeiro_agent`; +- publicação opcional de scores no Langfuse; +- persistência de histórico de avaliações no Oracle/ADB; +- acompanhamento de progresso via API ou CLI. + +--- + +## 3. Como ele se integra ao `agent_framework_oci` + +O `agent_framework_oci` é o runtime principal dos agentes. Ele executa o fluxo conversacional com LangGraph, supervisor, guardrails, judges, MCP tools, memória, RAG e telemetria. + +Durante a execução, o framework publica traces no Langfuse contendo: + +- `trace_id`; +- `session_id`; +- `message_id`; +- `agent_id`; +- `channel`; +- `business_context` canônico; +- eventos IC/NOC/GRL; +- spans de LangGraph; +- spans de guardrails; +- spans de judges; +- generations LLM; +- usage de modelo, quando disponível; +- `prompt_tokens`, `completion_tokens` e `total_tokens`, quando retornados pelo provider; +- `input_size`, quando emitido pelos spans do framework. + +O evaluator usa essa telemetria como fonte de dados. + +A integração principal acontece assim: + +```text +agent_framework_oci + ├── executa agentes + ├── resolve identidade via identity.yaml + ├── monta BusinessContext canônico + ├── chama MCP/RAG/LLM + ├── emite telemetria Langfuse + └── grava usage/model/tokens quando disponíveis + +agent_framework_evaluator + ├── lê traces no Langfuse + ├── aplica identity.yaml para normalizar identidade + ├── reconstrói ConversationRecord + ├── executa LLM Judge offline + ├── grava resultados no Oracle/ADB + ├── exporta TXT.GZ legado + └── opcionalmente publica scores no Langfuse +``` + +--- + +## 4. Arquitetura geral + +```text ++------------------------+ +| agent_framework_oci | +|------------------------| +| LangGraph | +| Supervisor | +| Guardrails | +| Judges online | +| MCP Tool Router | +| RAG | +| Memory / Checkpoint | +| Langfuse Telemetry | ++-----------+------------+ + | + v ++------------------------+ +| Langfuse | +|------------------------| +| Traces | +| Spans | +| Generations | +| Metadata | +| Usage / Tokens | ++-----------+------------+ + | + v ++------------------------+ +| agent_framework_ | +| evaluator | +|------------------------| +| Collectors | +| Identity Resolver | +| Conversation Records | +| LLM Judge | +| VLoop analytics | +| Repository Oracle | +| Legacy Exporter | +| API / CLI | ++-----------+------------+ + | + v ++------------------------+ +| Oracle ADB | +|------------------------| +| EVALUATION_RUN | +| EVALUATION_ITEM | +| EVALUATION_RESULT | +| EVALUATION_FINDING | +| EVALUATION_PROGRESS | +| EVALUATION_METRIC | ++-----------+------------+ + | + v ++------------------------+ +| Output | +|------------------------| +| TXT.GZ legado | +| API dashboard | +| Langfuse scores | ++------------------------+ +``` + +--- + +## 5. Componentes da solução + +### 5.1 CLI + +Arquivo principal: + +```text +evaluator/cli.py +``` + +Responsável por expor comandos como: + +```bash +python -m evaluator.cli init-db +python -m evaluator.cli show-config +python -m evaluator.cli run --source langfuse +python -m evaluator.cli run-agents --source langfuse +python -m evaluator.cli runs +python -m evaluator.cli progress +``` + +A CLI é a forma principal de operar o evaluator em modo batch. + +--- + +### 5.2 API + +Arquivo principal: + +```text +evaluator/api/main.py +``` + +Expõe endpoints HTTP para consultar progresso, runs e resultados. + +Exemplos esperados: + +```text +GET /health +GET /runs +GET /runs/{run_id}/progress +GET /runs/{run_id}/results +GET /runs/{run_id}/findings +``` + +A API permite construir uma interface gráfica simples ou integrar o evaluator com outros sistemas. + +--- + +### 5.3 EvaluationEngine + +Arquivo principal: + +```text +evaluator/engine.py +``` + +É o orquestrador central do evaluator. + +Responsabilidades: + +1. criar uma nova execução de avaliação (`EVALUATION_RUN`); +2. escolher o collector conforme `source`; +3. coletar conversas; +4. aplicar amostragem por agente; +5. inserir itens em `EVALUATION_ITEM`; +6. processar cada item; +7. chamar o LLM Judge; +8. salvar resultado de trace; +9. executar avaliação de sessão; +10. salvar resultado de sessão; +11. exportar arquivo legado; +12. marcar status final da execução; +13. emitir eventos de progresso. + +Fluxo simplificado: + +```text +run_agent() + ↓ +collector.collect() + ↓ +repository.insert_items() + ↓ +_process() + ↓ +judge.judge_trace() + ↓ +repository.save_trace_result() + ↓ +judge.judge_sessions() + ↓ +repository.save_session_result() + ↓ +export_legacy_txt_gz() +``` + +--- + +### 5.4 Collectors + +Diretório: + +```text +evaluator/collectors/ +``` + +Collectors são responsáveis por buscar conversas em uma fonte externa e convertê-las para `ConversationRecord`. + +Collectors típicos: + +```text +evaluator/collectors/langfuse.py +evaluator/collectors/agent_framework.py +evaluator/collectors/mock.py +evaluator/collectors/base.py +``` + +#### LangfuseCollector + +É o collector principal. + +Responsabilidades: + +- buscar traces no Langfuse; +- filtrar por período; +- filtrar por agente/alias; +- recuperar detalhes do trace; +- extrair input/output; +- reconstruir mensagens; +- coletar metadata; +- aplicar `identity.yaml`; +- montar `BusinessContext` canônico; +- preencher `ConversationRecord`. + +O collector deve normalizar dados para que o exporter não precise conhecer detalhes internos do Langfuse. + +--- + +### 5.5 Identity Resolver + +Diretório recomendado: + +```text +evaluator/identity/ +``` + +Arquivo principal: + +```text +evaluator/identity/resolver.py +``` + +O evaluator deve usar o mesmo conceito de identidade do `agent_framework_oci`, baseado no arquivo: + +```text +configs/identity.yaml +``` + +A função do `identity.yaml` é mapear campos variáveis de entrada para um modelo canônico: + +```text +customer_key +contract_key +interaction_key +account_key +resource_key +session_key +``` + +Exemplo conceitual: + +```yaml +identity: + version: 2 + keys: + customer_key: + sources: + - business_context.customer_key + - metadata.customer_key + - user_id + contract_key: + sources: + - business_context.contract_key + - metadata.contract_key + interaction_key: + sources: + - business_context.interaction_key + - metadata.ura_call_id + - metadata.message_id + - message_id + session_key: + sources: + - business_context.session_key + - session_id + - conversation_key +``` + +Com isso, o evaluator não fica preso a campos como `ura_call_id`, `call_id`, `message_id` ou `interaction_key` diretamente. Ele resolve tudo para `interaction_key`. + +--- + +### 5.6 Models + +Arquivo principal: + +```text +evaluator/core/models.py +``` + +Define os objetos centrais do evaluator. + +Principais modelos: + +```python +class ConversationRecord +class ConversationMessage +class TraceJudgeResult +class SessionJudgeResult +class CombinedJudgeResult +class RunStatus +class ItemStatus +``` + +#### ConversationRecord + +Representa uma conversa ou turno avaliado. + +Campos comuns: + +```text +trace_id +session_id +message_id +agent_id +channel +input_text +output_text +messages +metadata +raw +``` + +O campo `metadata` deve conter dados normalizados: + +```text +business_context +uraCallId +channelId +messageId +promptLength +``` + +O campo `raw` mantém o payload original para auditoria e fallback. + +--- + +### 5.7 LLM Judge + +Arquivo principal: + +```text +evaluator/judges/llm_judge.py +``` + +Classe principal: + +```python +TIMStyleLLMJudge +``` + +Responsabilidades: + +- carregar prompts de avaliação; +- montar prompt de trace; +- montar prompt de sessão; +- chamar LLM via client configurado; +- interpretar resposta JSON; +- retornar `TraceJudgeResult` e `SessionJudgeResult`. + +O judge avalia métricas como: + +```text +judgeScore +accuracyScore +alucinationScore +inferredCsiScore +resolution +conversationPrecision +rationale +``` + +O judge deve ser LLM-based, não determinístico. + +--- + +### 5.8 Prompts + +Diretório: + +```text +evaluator/prompts/ +``` + +Arquivos esperados: + +```text +trace_judge_prompt.md +session_judge_prompt.md +loader.py +``` + +O prompt de trace avalia uma resposta individual. + +O prompt de sessão avalia a conversa agrupada por `session_id`. + +Exemplo de saída esperada do LLM para trace: + +```json +{ + "judgeScore": 0.8, + "accuracyScore": 0.9, + "alucinationScore": 0.1, + "rationale": "Resposta aderente ao contexto e baseada em dados disponíveis." +} +``` + +Exemplo de saída esperada para sessão: + +```json +{ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "A conversa foi resolvida com informações consistentes." +} +``` + +--- + +### 5.9 LLM Client + +Diretório: + +```text +evaluator/llm/ +``` + +Arquivos típicos: + +```text +evaluator/llm/client.py +evaluator/llm/oci_openai.py +``` + +O evaluator deve usar o mesmo padrão de acesso a LLM do `agent_framework_oci`, preferencialmente via provider `oci_openai`. + +Variáveis comuns: + +```env +LLM_PROVIDER=oci_openai +OCI_GENAI_ENDPOINT=... +OCI_GENAI_MODEL_ID=... +OCI_GENAI_API_KEY=... +OCI_GENAI_COMPARTMENT_ID=... +``` + +O client precisa retornar texto bruto para o Judge interpretar como JSON. + +--- + +### 5.10 Repository / Oracle Store + +Diretório: + +```text +evaluator/persistence/ +``` + +Arquivos principais: + +```text +evaluator/persistence/oracle_store.py +evaluator/persistence/repository.py +``` + +O `OracleStore` cuida de: + +- conexão com ADB/Oracle; +- wallet; +- DSN; +- criação/ajuste de schema; +- execução thread-safe para chamadas assíncronas; +- prefixo de tabelas. + +O `EvaluationRepository` cuida de: + +- criar runs; +- gravar progresso; +- inserir itens; +- buscar próximos itens; +- marcar item como `PROCESSING`, `COMPLETED` ou `FAILED`; +- salvar resultados; +- salvar findings; +- sumarizar run; +- listar runs; +- consultar progresso. + +--- + +### 5.11 Legacy Exporter + +Arquivo principal: + +```text +evaluator/output/legacy_exporter.py +``` + +Gera o arquivo legado: + +```text +output/AGENTE__LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +Formato das colunas: + +```text +judgeScore +accuracyScore +alucinationScore +promptLength +loop +inferredCsiScore +resolution +conversationPrecision +uraCallId +channelId +sessionId +messageId +``` + +Exemplo: + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"TOTAL"|;"19" +``` + +#### promptLength + +O campo `promptLength` deve usar esta prioridade: + +1. `prompt_tokens` / `promptTokens` / `input_tokens` / `inputTokens` nas observations do Langfuse; +2. `usage.input` ou `usageDetails.input`; +3. `metadata.input_size` emitido pelo framework; +4. fallback para tamanho textual de `input_text`, `output_text` e `messages`. + +Exemplo: + +```text +promptLength = 732 +``` + +#### loop + +O campo `loop` usa o detector VLoop. + +```text +0 = sem loop detectado +1 = loop detectado +``` + +--- + +### 5.12 VLoop Analytics + +Arquivo principal: + +```text +evaluator/analytics/vloop.py +``` + +Responsável por detectar repetição/loop conversacional em padrão semelhante ao guardrail VLoop do `agent_framework_oci`. + +A função normalmente exposta é: + +```python +vloop_flag(raw) -> int +``` + +Ela retorna: + +```text +0 quando não há evidência de loop +1 quando há repetição suspeita +``` + +--- + +### 5.13 Langfuse Score Publisher + +Arquivo principal: + +```text +evaluator/publishers/langfuse_scores.py +``` + +Responsável por publicar scores de avaliação de volta no Langfuse, quando habilitado. + +Variável de controle: + +```env +PUBLISH_LANGFUSE_SCORES=true +``` + +Quando desabilitado, o evaluator apenas grava no banco e exporta arquivo. + +--- + +## 6. Estrutura de diretórios + +```text +agent_framework_evaluator/ +├── configs/ +│ ├── identity.yaml +│ └── judge/ +│ └── agents.yaml +├── docs/ +├── evaluator/ +│ ├── __init__.py +│ ├── cli.py +│ ├── engine.py +│ ├── api/ +│ │ └── main.py +│ ├── analytics/ +│ │ └── vloop.py +│ ├── collectors/ +│ │ ├── base.py +│ │ ├── langfuse.py +│ │ ├── agent_framework.py +│ │ └── mock.py +│ ├── config/ +│ │ ├── settings.py +│ │ └── agents.py +│ ├── core/ +│ │ └── models.py +│ ├── identity/ +│ │ └── resolver.py +│ ├── judges/ +│ │ └── llm_judge.py +│ ├── llm/ +│ │ ├── client.py +│ │ └── oci_openai.py +│ ├── output/ +│ │ └── legacy_exporter.py +│ ├── persistence/ +│ │ ├── oracle_store.py +│ │ └── repository.py +│ ├── prompts/ +│ │ ├── loader.py +│ │ ├── trace_judge_prompt.md +│ │ └── session_judge_prompt.md +│ └── publishers/ +│ └── langfuse_scores.py +├── output/ +├── Dockerfile +├── docker-compose.yml +├── pyproject.toml +└── README.md +``` + +--- + +## 7. Configuração + +### 7.1 Arquivo `.env` + +Exemplo: + +```env +# Oracle / ADB +ADB_USER=ADMIN +ADB_PASSWORD=your_password +ADB_DSN=oradb23ai_high +ADB_WALLET_DIR=/path/to/Wallet_ORADB23ai +DB_TABLE_PREFIX=AGENTFW_ + +# Langfuse +LANGFUSE_ENABLED=true +LANGFUSE_HOST=http://localhost:3005 +LANGFUSE_PUBLIC_KEY=pk-lf-... +LANGFUSE_SECRET_KEY=sk-lf-... +PUBLISH_LANGFUSE_SCORES=false + +# LLM +LLM_PROVIDER=oci_openai +OCI_GENAI_ENDPOINT=https://... +OCI_GENAI_MODEL_ID=... +OCI_GENAI_API_KEY=... +OCI_GENAI_COMPARTMENT_ID=... + +# Evaluator +EVALUATOR_OUTPUT_DIR=output +EVALUATOR_BATCH_SIZE=10 +EVALUATOR_MAX_ATTEMPTS=2 +EVALUATOR_AGENTS_CONFIG=configs/judge/agents.yaml +IDENTITY_CONFIG_PATH=configs/identity.yaml +TRACE_PROMPT_PATH=evaluator/prompts/trace_judge_prompt.md +SESSION_PROMPT_PATH=evaluator/prompts/session_judge_prompt.md +``` + +--- + +### 7.2 Configuração de agentes + +Arquivo: + +```text +configs/judge/agents.yaml +``` + +Exemplo: + +```yaml +agents: + - agent_id: telecom_contas + enabled: true + aliases: + - telecom_contas + - billing_agent + - financeiro_agent + percentage: 1.0 + + - agent_id: retail_orders + enabled: true + aliases: + - retail_orders + - orders_agent + percentage: 1.0 + + - agent_id: financeiro_agent + enabled: true + aliases: + - financeiro_agent + percentage: 1.0 +``` + +O campo `aliases` é importante porque o Langfuse pode registrar o agente de formas diferentes, por exemplo: + +```text +agent_id = telecom_contas +route = financeiro_agent +agent = financeiro_agent +``` + +--- + +### 7.3 Configuração de identidade + +Arquivo: + +```text +configs/identity.yaml +``` + +O evaluator deve usar o mesmo padrão do framework. + +Exemplo: + +```yaml +identity: + version: 2 + keys: + customer_key: + sources: + - business_context.customer_key + - metadata.customer_key + - user_id + + contract_key: + sources: + - business_context.contract_key + - metadata.contract_key + + interaction_key: + sources: + - business_context.interaction_key + - metadata.ura_call_id + - metadata.message_id + - message_id + + session_key: + sources: + - business_context.session_key + - metadata.session_key + - session_id + - conversation_key +``` + +O campo `interaction_key` é usado para preencher o `uraCallId` no export legado. + +--- + +## 8. Como executar + +### 8.1 Instalar dependências + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +Se estiver usando Conda: + +```bash +conda activate py313 +pip install -e . +``` + +--- + +### 8.2 Validar configuração + +```bash +python -m evaluator.cli show-config +``` + +Saída esperada: + +```text +{ + "env_path": ".../.env", + "adb_dsn": "oradb23ai_high", + "wallet": ".../Wallet_ORADB23ai", + "langfuse": true, + "publish_langfuse_scores": false, + "llm_provider": "oci_openai", + "agents_config": "configs/judge/agents.yaml" +} +``` + +--- + +### 8.3 Criar/validar schema + +```bash +python -m evaluator.cli init-db +``` + +Saída esperada: + +```text +{'status': 'OK', 'message': 'Evaluator schema checked/created successfully.'} +``` + +--- + +### 8.4 Rodar avaliação por período + +```bash +python -m evaluator.cli run \ + --period-start 2026-06-11T00:00:00 \ + --period-end 2026-06-12T00:00:00 \ + --source langfuse +``` + +--- + +### 8.5 Rodar avaliação por agentes configurados + +```bash +python -m evaluator.cli run-agents --source langfuse +``` + +Saída esperada: + +```text +[ + { + 'status': 'COMPLETED', + 'run_id': '...', + 'total_items': 19, + 'completed_items': 19, + 'failed_items': 0, + 'evaluations': 19, + 'avg_score': 0.72, + 'agent_id': 'telecom_contas', + 'output_file': 'output/AGENTE_telecom_contas_LLM_JUDGE_20260612.TXT.GZ', + 'uploaded_to': None + } +] +``` + +--- + +### 8.6 Consultar progresso + +```bash +python -m evaluator.cli progress +``` + +Ou via API: + +```bash +curl http://localhost:8001/runs//progress +``` + +--- + +### 8.7 Ver arquivo exportado + +```bash +gzip -cd output/AGENTE_telecom_contas_LLM_JUDGE_20260612.TXT.GZ +``` + +Exemplo de linha válida: + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"TOTAL"|;"19" +``` + +--- + +## 9. Banco de dados + +### 9.1 Tabelas principais + +#### EVALUATION_RUN + +Armazena uma execução de avaliação. + +Campos principais: + +```text +RUN_ID +PERIOD_START +PERIOD_END +SOURCE +AGENT_ID +STATUS +TOTAL_ITEMS +PROCESSED_ITEMS +FAILED_ITEMS +LAST_HEARTBEAT_AT +CREATED_AT +UPDATED_AT +ERROR_MESSAGE +``` + +--- + +#### EVALUATION_ITEM + +Armazena cada conversa/turno coletado. + +Campos principais: + +```text +ITEM_ID +RUN_ID +TRACE_ID +SESSION_ID +MESSAGE_ID +AGENT_ID +CHANNEL +STATUS +ATTEMPT_COUNT +RAW_JSON +CREATED_AT +UPDATED_AT +ERROR_MESSAGE +``` + +--- + +#### EVALUATION_RESULT + +Armazena resultados de trace e sessão. + +Campos principais: + +```text +RESULT_ID +RUN_ID +ITEM_ID +TRACE_ID +SESSION_ID +AGENT_ID +JUDGE_TYPE +JUDGE_NAME +JUDGE_SCORE +ACCURACY_SCORE +ALUCINATION_SCORE +INFERRED_CSI_SCORE +RESOLUTION +CONVERSATION_PRECISION +RATIONALE +RESULT_JSON +CREATED_AT +``` + +`JUDGE_TYPE` pode ser: + +```text +TRACE +SESSION +``` + +--- + +#### EVALUATION_PROGRESS_EVENT + +Armazena eventos de progresso da execução. + +Exemplos de stage: + +```text +RUN_CREATED +COLLECTING +COLLECTED +SAMPLED +ITEMS_INSERTED +BATCH_STARTED +ITEM_COMPLETED +ITEM_FAILED +SESSION_JUDGE_COMPLETED +EXPORTED +COMPLETED +PARTIAL +``` + +--- + +## 10. Como os códigos funcionam em conjunto + +### 10.1 Fluxo completo de execução + +```text +CLI run-agents + ↓ +carrega configs/judge/agents.yaml + ↓ +para cada agente habilitado + ↓ +EvaluationEngine.run_agent(agent) + ↓ +cria EVALUATION_RUN + ↓ +LangfuseCollector.collect(...) + ↓ +IdentityResolver.resolve(...) + ↓ +ConversationRecord + ↓ +EvaluationRepository.insert_items(...) + ↓ +EvaluationEngine._process(run_id) + ↓ +TIMStyleLLMJudge.judge_trace(record) + ↓ +LLMClient.complete(prompt) + ↓ +save_trace_result(...) + ↓ +TIMStyleLLMJudge.judge_sessions(records) + ↓ +save_session_result(...) + ↓ +export_legacy_txt_gz(...) + ↓ +COMPLETED +``` + +--- + +### 10.2 Papel do collector + +O collector é responsável por transformar dados externos em dados canônicos. + +Ele deve esconder diferenças entre fontes como: + +```text +Langfuse +agent_framework database +mock data +``` + +A saída sempre deve ser: + +```python +ConversationRecord +``` + +--- + +### 10.3 Papel do judge + +O judge recebe um `ConversationRecord`, monta um prompt e chama o LLM. + +Ele não deve conhecer Oracle, Langfuse, export legado ou API. + +Ele só avalia. + +--- + +### 10.4 Papel do repository + +O repository é a camada de persistência. + +Ele não deve conter regra de negócio de avaliação. + +Ele apenas grava, busca e atualiza dados. + +--- + +### 10.5 Papel do exporter + +O exporter transforma dados persistidos em arquivo legado. + +Ele não deve resolver identidade de forma complexa. + +O ideal é que ele leia campos já normalizados: + +```text +metadata.business_context.interaction_key +metadata.channelId +metadata.messageId +metadata.promptLength +``` + +No entanto, para resiliência, ele também pode consultar `RAW_JSON` como fallback. + +--- + +## 11. Regras importantes de desenho + +### 11.1 O evaluator não deve ficar chumbado para um agente + +Evite lógica como: + +```python +if agent_id == "telecom_contas": + ura_call_id = metadata["ura_call_id"] +``` + +O correto é usar `identity.yaml`. + +--- + +### 11.2 O exporter não deve conhecer detalhes internos do Langfuse + +Evite acoplamento excessivo a caminhos como: + +```text +raw.detail.observations[0].metadata.ura_call_id +raw.trace.input.business_context.interaction_key +``` + +Isso deve ser resolvido no collector. + +--- + +### 11.3 `promptLength` deve vir de tokens quando possível + +Prioridade recomendada: + +```text +1. prompt_tokens / promptTokens +2. input_tokens / inputTokens +3. usage.input / usageDetails.input +4. metadata.input_size +5. tamanho textual de input/output/messages +``` + +--- + +### 11.4 `uraCallId` deve vir do BusinessContext + +O campo legado `uraCallId` deve ser mapeado para: + +```text +business_context.interaction_key +``` + +Esse é o nome canônico do framework. + +--- + +### 11.5 `sessionId` deve vir do BusinessContext + +O campo legado `sessionId` deve ser mapeado para: + +```text +business_context.session_key +``` + +Não confundir com a chave composta completa: + +```text +default:telecom_contas: +``` + +O evaluator pode guardar a chave completa, mas o export legado normalmente deve usar o identificador de sessão limpo. + +--- + +## 12. Testes recomendados + +### 12.1 Teste de configuração + +```bash +python -m evaluator.cli show-config +``` + +Validar: + +```text +ADB_DSN +Wallet +Langfuse enabled +LLM provider +Agents config +Identity config +``` + +--- + +### 12.2 Teste de banco + +```bash +python -m evaluator.cli init-db +``` + +Depois validar tabelas: + +```sql +select table_name +from user_tables +where table_name like 'AGENTFW_EVALUATION%'; +``` + +--- + +### 12.3 Teste com mock + +```bash +python -m evaluator.cli run --source mock +``` + +Use esse teste para validar schema, judge e export sem depender do Langfuse. + +--- + +### 12.4 Teste com Langfuse + +```bash +python -m evaluator.cli run-agents --source langfuse +``` + +Validar: + +```text +total_items > 0 +completed_items > 0 +failed_items = 0 +evaluations > 0 +output_file preenchido +``` + +--- + +### 12.5 Teste do export + +```bash +gzip -cd output/AGENTE_telecom_contas_LLM_JUDGE_YYYYMMDD.TXT.GZ +``` + +Validar colunas: + +```text +judgeScore preenchido +accuracyScore preenchido +alucinationScore preenchido +promptLength maior que 0 +loop 0 ou 1 +inferredCsiScore preenchido +resolution 0 ou 1 +conversationPrecision 0 ou 1 +uraCallId preenchido +channelId preenchido +sessionId preenchido +messageId preenchido +``` + +--- + +## 13. Troubleshooting + +### 13.1 `promptLength` sai 0 + +Causas comuns: + +- `find_prompt_tokens` não foi incluído no arquivo; +- `promptTokens` está zerado no Langfuse; +- `input_size` não está sendo percorrido; +- `RAW_JSON` está vindo como string não convertida; +- exporter antigo ainda está rodando; +- `except Exception: pass` está mascarando erro. + +Debug recomendado: + +```python +print("PROMPT_LENGTH", extract_prompt_length(raw)) +print("RAW_TYPE", type(raw)) +print("RAW_KEYS", list(raw.keys())[:20]) +``` + +--- + +### 13.2 `uraCallId` sai vazio + +Causas comuns: + +- `identity.yaml` não está sendo carregado; +- collector não está copiando `business_context` para `metadata`; +- `interaction_key` não existe no trace; +- exporter não usa `business_context.interaction_key`. + +Validação: + +```sql +select RAW_JSON +from AGENTFW_EVALUATION_ITEM +where MESSAGE_ID = ''; +``` + +Procurar: + +```text +interaction_key +ura_call_id +business_context +``` + +--- + +### 13.3 `ORA-00904 invalid identifier` + +Geralmente indica schema antigo. + +Exemplos já encontrados: + +```text +ORA-00904: UPDATED_AT invalid identifier +ORA-00904: REASONING invalid identifier +ORA-00904: JUDGE_TYPE invalid identifier +``` + +Correção: + +```bash +python -m evaluator.cli init-db +``` + +Se a tabela já existir sem a coluna nova, o `_init_schema` precisa executar `ALTER TABLE ADD` de forma idempotente. + +--- + +### 13.4 `ORA-00054 resource busy` + +Indica lock em tabela. + +Causas comuns: + +- API rodando enquanto `init-db` tenta alterar schema; +- outro processo usando a tabela; +- transação aberta no SQL Developer. + +Correção: + +1. parar API/CLI; +2. fechar sessões abertas; +3. executar novamente `init-db`. + +--- + +### 13.5 `OCI LLM 401` + +Indica problema de autenticação no LLM. + +Validar: + +```env +OCI_GENAI_ENDPOINT +OCI_GENAI_MODEL_ID +OCI_GENAI_API_KEY +OCI_GENAI_COMPARTMENT_ID +``` + +Também confirmar se o evaluator está lendo o `.env` correto: + +```bash +python -m evaluator.cli show-config +``` + +--- + +### 13.6 `Entity with key ${OCI_GENAI_MODEL_ID} not found` + +Indica que o valor literal `${OCI_GENAI_MODEL_ID}` chegou ao provider. + +Causas comuns: + +- variável não expandida; +- YAML usando `${OCI_GENAI_MODEL_ID}` sem interpolação; +- `.env` não carregado; +- configuração do LLM client não resolve placeholders. + +Correção: + +- colocar o model ID real no `.env`; +- garantir interpolação em `settings.py`; +- validar com `show-config`. + +--- + +## 14. Checklist de validação final + +Antes de considerar o evaluator pronto, validar: + +```text +[ ] init-db executa sem erro +[ ] show-config mostra .env correto +[ ] Langfuse retorna traces +[ ] run-agents coleta itens por agente +[ ] LLM Judge responde JSON válido +[ ] EVALUATION_RESULT grava TRACE e SESSION +[ ] progress mostra eventos úteis +[ ] export TXT.GZ é gerado +[ ] promptLength > 0 +[ ] uraCallId preenchido +[ ] sessionId preenchido +[ ] messageId preenchido +[ ] loop preenchido com 0 ou 1 +[ ] arquivo termina com TOTAL +[ ] scores podem ser publicados no Langfuse quando habilitado +``` + +--- + +## 15. Exemplo de resultado final validado + +```text +"0.8"|;"0.9"|;"0.1"|;"732"|;"0"|;"0.5"|;"1"|;"1"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"6d7e85b0-ddd0-4f23-a372-30e754a4491a" +"0.9"|;"1"|;"0"|;"642"|;"0"|;"0.5"|;"1"|;"1"|;"5ab3ea80-7428-402f-98ec-04e7cd5327e4"|;"web"|;"eba23248-e038-4d33-bc2c-6465ef677d07"|;"5ab3ea80-7428-402f-98ec-04e7cd5327e4" +"TOTAL"|;"19" +``` + +Esse resultado indica: + +- Judge funcionando; +- prompt tokens extraídos corretamente; +- VLoop sem ocorrência; +- métricas de sessão preenchidas; +- identidade canônica funcionando; +- export legado no layout esperado. + +--- + +## 16. Resumo executivo + +O `agent_framework_evaluator` é a camada batch/offline de avaliação do ecossistema `agent_framework_oci`. + +Ele consome a telemetria gerada pelo framework, aplica um Judge LLM com regras de avaliação, persiste resultados em Oracle/ADB, gera arquivo e pode republicar scores no Langfuse. + +A arquitetura correta separa responsabilidades: + +```text +Collector normaliza dados +IdentityResolver resolve identidade +Judge avalia conversa +Repository persiste +Exporter gera legado +API/CLI operam a solução +``` + +Com isso, o evaluator fica genérico para múltiplos agentes e evita acoplamento direto a formatos específicos de trace ou payload. diff --git a/configs/.DS_Store b/configs/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..80b3397b7b7b1d9e6892d8011b8526d912d7768f GIT binary patch literal 6148 zcmeHKK~BRk5FD2x1;L?5xcLe62cZft=mS6{rGOG80`;Cx@Fq@hfOqf*X1!L06H0Fg z&~CLm_IlTGCTZ3N;AZP`3QPcGRKd{!%@-o$q5~O4j|{pn$Ff>Y=T}YL4r0ymI~kB? zw~srlQDTMO^UI#o(Yh&$Mb#|mmAuE7_ouUdM7blPz4`oxoH89psQ9aKffn~@aE)8` zmw4#CO|Rd@_SSatrfb_eciki_-5H;*TmCO5#(*(k4E$LJShGd4Lq)U3fH7bUY#5O5 zgQp7S5j(~3>7WTm0OFYLAZ&-d5z&i-m`Cgs>7h7HC2FdRM+~Rw^!pf>N9+_e9WGp% z^TaE=ctUY;XXbnehszbs8Uw~a%D|p>CsO~SM}G2<=8+B uskNSZNfnW}PH~gMMkvLMl~R064MM+{1~HG=Dbho+jX, "resolution": <0|1>, "conversationPrecision": <0|1>, "rationale": ""} + + rationale deve ter no máximo 200 caracteres. diff --git a/configs/judge/trace_metrics.yaml b/configs/judge/trace_metrics.yaml new file mode 100644 index 0000000..6e73b5a --- /dev/null +++ b/configs/judge/trace_metrics.yaml @@ -0,0 +1,16 @@ +trace_metrics: | + Você é um avaliador imparcial de respostas de agentes conversacionais. + Vai receber o HISTÓRICO da conversa (últimos turnos), a MENSAGEM DO + USUÁRIO e a RESPOSTA DO AGENTE. Sua tarefa é atribuir três notas + numéricas entre 0.0 e 1.0: + + - judgeScore: qualidade global da resposta (clareza, utilidade, tom). + - accuracyScore: factualidade da resposta frente ao contexto fornecido. + - alucinationScore: grau em que a resposta contém afirmações NÃO + suportadas pelo contexto (alto = mais alucinação = pior). + + Retorne SOMENTE um JSON válido, sem nenhum texto adicional, no formato: + + {"judgeScore": , "accuracyScore": , "alucinationScore": , "rationale": ""} + + rationale deve ter no máximo 200 caracteres. diff --git a/configs/llm_profiles/llm_profiles.yaml b/configs/llm_profiles/llm_profiles.yaml new file mode 100644 index 0000000..7975152 --- /dev/null +++ b/configs/llm_profiles/llm_profiles.yaml @@ -0,0 +1,11 @@ +profiles: + judge: + provider: oci_openai + model: meta.llama-3.3-70b-instruct + temperature: 0 + max_tokens: 900 + mock: + provider: mock + model: mock-judge + temperature: 0 + max_tokens: 300 diff --git a/evaluator/.DS_Store b/evaluator/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..8b2d75b9270435a913fe1612d538dec207f2d0c5 GIT binary patch literal 10244 zcmeHMeQX@X6`!{@&TgEI*GWurj?-*#8X|C%a}p8~7kd5(#c`+u{&ozPyW7QH*xNmJ z_s$M(Bv%DmwGu@Y<*P-gMWX*uN-8S#3lY8%h_5J$1W2?9B#Dl70-rBFS+DvIE@T!s>`KOq^C z!fS@;Bw2xu8X^xw9*8^;c_8vYXC1^8%S zy$=URORK2i!5kgN9n~EdWqTq{9e^45NYlCv7i_sWe#yS?_FZ{s_>|JTYW12HS)RnX zusq{>Lw81I4b7XGNVyr^(hO^QtYElC+SG@wR90_>D<$dF>gH8StTomqD(&KUK5g8T zH@u^)Ro)oeBq@Iqr!*vUm95bcHICmH`?e?^rtQ6sj(6THZ;f3fDJgN-MW65?w#nNi z#g&F{#DL*lcgVX$MUzI%R9ZKA(;j)Rs0>JB1^$G*@jLS6Vl#9|#t$bG6xgezdM2>Hkn6D*;ZoxK^tz;kR zA*1*TbIA|L?c`qaFnN-^NM0gul6T36G_p=21+vI(+NInL@U|$XEFvL6H66k_%4DBHpfpM6CNepitreTJKxQ78g3v=*8 z_z?#AZ5ZZvz%SrV_$Ay055RBWw=C2jVWIv<_#6Bko`4hZ6ubm4!>jNbybdSfEqD(W z;Xh@e{QY9aatZ$}59Qz*zBu}q#TLGXhr&I$cGtp$)@*VIH5VGAFyvUKZPC31O|yEo zo;ySjFK?^f4C;d1(rR0$zt>sn8{}0@ALgXu&^U7j(QvP0vE83Xj)v{aA5_Bxc?>$A zu3f`@R54bN&zW#lcr}n4NFJF@Be$|0Y5SwdDWnhb*?yl?+>4fBN;|))r&6Cc5OOv!g?Nfz8u0 zK*G;bBC60h2*5FetVF+4}m8BAmJQ2)_?aVyRdO;^;y z1-V^#eCcdeb$G3OmkV5C3zl#E&w%Lsk6w??|NqbD|1SYp CdY9z@ literal 0 HcmV?d00001 diff --git a/evaluator/__init__.py b/evaluator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evaluator/__pycache__/__init__.cpython-313.pyc b/evaluator/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fd2e75673dce5d378ff540aad3b49f29dbe6125c GIT binary patch literal 206 zcmey&%ge<81oKk0vq1D?5CH>>P{wB#AY&>+I)f&o-%5reCLr%KNa|LuerR!OQL%n< zQD$*TW@27`Mt*TdW_DtEqP|Ohd0tL_VoI@oVtQ&`NqkySVs2`Aeo=OOYFT1VX<|u! w5gr+Yn)vw4yv&mLc)fzkTO2mI`6;D2sdh!IK-)pCD+Vz>GBYwV7BK@^0P}-88vpSvC#n>`drPXG)&RQ-^&E%ua_#X$7_SE_6H7ozBoFQ^K%6edza{t1HQl zo$W)9$LF5&edqT(-@l$$Ryqk3`O5B#SE~v65A0Y)G#lLWJvDN?=x!#o>1^ZJ zz7p1I1uGA5W;Kt|=ScvdQS$;^>xOYp|}FTof;I z#S^t|GONBgYg@1_*syNawoPU=7H9c_?ZKvXv)VS9wY6+kN3eO_EZ^Dop!eVz;2Q@a zXB#_;)bx=_ev~TC0QQ7nm4T!@qbqR}gqtQcIUbcK z6*a9xcgw(KIhkQ+7*Hn@ssgaO5^PMF(ey-GqjLav=t?@BP$%{I&R(4FJ4WS{GON*P zU+Bb0r=FfmD!$?25#L}YKB@Q$t7sIaic?B*CRX4u##|^r0wLVx;xgZC$ zbf{c}4Z&&*q-%ZYAYraVRqHKYA$~p(D<4|{og27C2sS>z+uc{v06m3AVJ>RBx1K<8;lZ)qe?TyA`DN%iZ%0q&Fu3e#fSJYR{>r22}(g* z&>nR32q`hlhlQ}XUkFw(5|NMuWFXe+CqU^;!3{2Rp9=xWO;iq6tSnoq&(d; z(L{W{Ei$ep&Z>?+XCjW$x z5(nbD_i$rHE-D?@n#Z_;|IVumCH@Qx8{V{HgPX$JgCTKlim|KWpIz6r>FnaVsuZ$~ z6{Vo;PL4AJVXle<-C@3p>=S?=D@78l>H!X&gg@X%m2q2jfY!n^+60wB40{SxAu!6Z zuR)E;nY0#7sEKq`k4-5lnbyM~K)964z)sO*B9%xRF2*d;L|RGd6gl2-#x+$j0rUJp z-_t7q^b52RaTP^YWjlfb7ABQ6twNj!_Dx5T^=B-dEP#?WY?&EU;{FOlVjG)h6;k@7 zA?m6;W3DXXu7;yH6}dO97$S;|jui5-vaqmVOm&BKh3efgn$Xh;S=FXAeJU|6&&u8X z+N_GoTrhgQ$w~ci7hDvqO{pIn zuY*(Sd8mH>lq`^yj-CJL|9k)Kck&$vmOI|LTYvY;FWQ!*rK#)HD^0CmReV`-Yv=8b zd{gI=w9?@HB5^%&s{;BBolC+>ZPU%BFaF|j`+;vVx%QFe_Gqp>n!TW8C)C^pEqnFD zEOgtiEqkvm^*?TIy&3;%>dUEHSM%ErEN?q_*OPC4Yq|Nz-AhXYkGHnm41IOx%QLr< z`PO~Qt#97Bl;7I3)W1^K^2O-&(OdnuGY`TKj%S16?5X#%;rFwHk?h$E*>e}NgYweo zujrn>QFUAAi5qV%}3{%5Hh+CR)^e-O;? z3_Uy*&7YDV#!{<^<-aB5Bp=}cc$UA6z?1we0_XTa0eW-%umHWP5dlV= z6y6h7JX@DN9XU_OtwVP%edEk}IpmO( zWb|hL?cO^l?_OLS&ACsk)NXz0#r2-CRi8DGhCMGE5%7Wm{zu;nSNZE+N8fAQy{a93 zjl#WsTwlHI-u^~t-*-6ye!rf>cB8Xzhy8xL9Ux#D?6N+m#uAz#0M-0FMY~JiK-!?_ zU+|eEtz1J&dJs_H>J!}y)$bR`3%*)v{?08$?-?)-@`Ld!9C_%`)10GRH zcgDwK*tH7kZLr1M^S^_}=cMFgfY-tIa4K4ng1xn-;CCZ^k7vdLwtC|^MSsK`hjnRA zq^NC!LQo7!T}3|zHLKqiI7VAwg29gt&F?R|Yd%wh`r=fZnNi|CS&jRs5~B*}sJKth z#A2X#Co;+8T-VpUVME=O(J9i)FPYjA^qEE#!-Xpp^s(Vyqn(kl(JiFtU!p}WyoO28 zXndUZ1F!@Bb$pQp@;g`Co%)>XhYS5HqT}Pi9}O;x%{j68q1aOFdvl`qq1gIZtXK+V zUpGmtX61(v({n_nr<(!8eiEo>Cap_aq zY-dbUClZru*bcW~Y_|jiEx%96v51H;hLtfq%eoTeIzeGshP-SnYPEHx49jBVwdgXe zdo2cgWmr7(x>APSvJM5^GHlKFV$ndYD`j&V)}f#q*=oVc#x+~8(su*;pb)JqLB6Z# z5;0*D0`p>*qF(m93fWSS)H87=!(%elNnEfv58s` z5F`*P-Eb$9DfBunC*oiUT!r35BB>ZPS}YL-UqDVo$7NlKW@yr=F2Pb7gp@{02_VlT zqSMM;)MTa%RpLe^^Ck2G$Y0HP))4eGrKlDSH|%^-iGlB87jum2ZRC}I^q z(|`WvvUhLJyZ6o;Id4zC@!&77EFV9eJAV3MY26mM*}b;$Pxtf(HwQ9{?VO z#0~(D3NbWW_yJ%>xQf6Gq@N&q}AjY|N;4*+MSYY3c` zMr{DZ4*)OOGWfY{`_Kls%eHszfWr?Mpi8!scEH8$Nju=;_G^eMd|XEyF9?x(9VR(? z;Lfq~cmhS5DT{xG#(8p{MaK|?i^Yhf7EX(rnH7jVA+8DY`$aQEw}e2IM)yXFwrs4Z zUaWbj!%MM)WtldzbRiOgm^CCED6CQ1p=8Cvk_BZV1>t`2fUqHIG&nT(94*FSv0MX& zA>fCA3NF09gap`B3~|=BEQ|6iEq&4xJK1GUEy-M6C%e z9#zyh2#DEcF)CO`Edv3qj->a2KVR_U9EXx}YCJ92$Wv>gT!7I}@vBKk-2Ss{-w;4wO8VaglH4c?g zi8iDdy3qXqW4`7EC_d8^n+xG;Xdd-LQ2l9v{HML{v1i+I+ts}1!%u9#^E7?o{i*k1 zYiGW-d%1OAu65s?;cs?5n8==r+xrs94ZVzk!_d4)BweD(EIEi4=!yBuA@ zE`m-fOCMFSa9s+p7~3yG_79}Mu%!}e2EtsMu7JUb8?IC$Mm4j8zF=IQ(?Q6JT zKVl*Lq1#|z(4lMTV0!F35XG~!6EF#;l5`e5T1pZx!$Np;;B^#a=2@6Gf7r0=?m(`g zZ@D3qYXHq1$wJ*QlXqTvBvM8U2!JxGzQ&3n&oOGK(D?tRMk!%zQNsEbd{i?a+?W`K zQwE2WWxe2+112VAp&Pvh=VjFpH1t=X+K?0@xrUJkO1>eKcZMH{r%W<9qEQr|XcX9S z7-j}SApCJ$P{b-$Y%$ghVHYjMADxhM^L^k8Kst?1Aruw1wMGDpUuit@^Sl%MSZh2k zKtICq+|xSzh-P+?t^Oe*UqvWy>I2s9Fr3loL?#WMax_Zel?rmh4k!`rezh^<@vWs{g|)3;a!jXQ3b9iQpf;H&Vb~${nS!{VMd2(sW$v@-6QIDGX)Ank z1Kv)BpMiiN#}HBSz8!}k;K(GE!}K~J&{Nafpu0=L3m1poj5 literal 0 HcmV?d00001 diff --git a/evaluator/__pycache__/engine.cpython-313.pyc b/evaluator/__pycache__/engine.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..17434e3b34137240e761955065b54c797597d974 GIT binary patch literal 12636 zcmb_CZEzdcad*Js`v{N#KR}QaA%2OZBv9X!D47;TQY1lq|g6rbyEZ=hX{((vBH4c#VS5YZ6RevtTA=x6cgMySmLz{c7mG*9o|x*l;GyUGHbYZ>>;EaND5UyIxrDrjDDB*B!4vZrq{K!9d&S<0g(^ zoMy9uqLfmh!AaFoT(OJdN_J|L`4AeXH7)i}Fw;WImS8j*iwEQ3SX3Nt(a6ThV4RPK zXL#9!aFUP2gDqNF+Z8=0o4TG3hU3AL5neWS2O|;0fJ9k$EP9p~L_(xH7K!kocuatb zl0(7h^wgZl=N49UP4m(CK_NK9KOGa!JxpLtOk^gg$U8SVjhvS9&&GN3Ldo~!EBmDdL*_apx{?9>qxjFz2=<|o?PDaAwDP9OE6RAM0@}M7_2IM8`I3>^= z1(HRBR9QHd)4HhB%xM~AtBA95Ogg9IY@7~pLC+O)C7eE8Zh)MfGo*7y&cPW;D-&1B znFwa)%z_10uuLw_a@`x94oCS<9KgbGD9sV!3%>!#(+|!Q?)>y)bTg@-s7Vt=xv2^G zHk8ECoMsR0ry`k>ad#vwGZZxeGNq)Q=_j;Ie_>E-DNw6d%ciLo?eIZdli*xD76?bf z@qidQ#m@v=3Q%cl9sH%thy*5 zq>RW%rUVx>5UK!(I1z5QTRzj~7ePMTLPA)KhlA1BshD^wd?xsGu&pQdbTkqRPKs?o z6wAPrDwhFX5s8@a6*bv5JXrwG>^a#OPzD%~EA!XKqgLS<#5MrmqE=iDDc9)j=6%b2 zqIvWJ^Ov@j_1kV6*WE2^OqOkcy!me3?qprZ1?HE=+6V9o70S#yWN?nZi|g9}t5y%| z`4Z)4e3=!*_DX`)rPLk3P5CKbW@XhD6y?K>Bro}Czs9fiGdt704#3`d3MTd}{gQTA z$3l;43r^!EbE%%Y#UCk-aI?AtC+h*v3 z{Q32(d7Gy2&qxjX;b&^sPSGHVvR;^r2EvmqjL-liWt|v@#g`2;AaewnB`MG5A<_leSZRIIj-G{ci`K>8iW5U+B@U=Ide)Va|)_6}xv2~x; z0O3#FRCC+!#X4Z@xg7_J>7URCSmvi)MSxyo=mS>fn$C#0l?E8f6+rw_$u{*2ARxjt zM|X@ghcg^Nd0zrQ``>#1j15s@mSJQd%Jm0vU#txelHYAWlc#- z)9to{$+mu}xgVR1B`sC254|=b)oj0ObxM_kN$VlWcu2tI3*~;ppeLy3agS++sZ8%N zln)B?lP|Lr*_!eC@h)F_@5q)?)LQ#VRKISm9YT*9%1;-fW~H{TQR8Pg?H=9godI?s z99Ld~eNNql3~TIDdXC|AJJTaAK-b{Q=)x@V@p|AJ7e6ogb)0?=lTRgTT(fTjXPmI9 zMEpjTTPwwxwrjtFE6RsDG%t<+sn4EQ1X)!}d<7R~&aws<>~<}Tv;bAEX!SUW9$L)+ z64s8gq_9Gq{EFn|tZPUqwMG6j$oe_RmPcNDfG^jsNN?q~C6F+-87NY7#(HLk4isukjjb9xw2Qwx?*lz;NF~XZmaeCmwew=ZJ;JpCaY8U?dzAc~RCy!e}m-(r@?nJT#|Y z<|p0foo>Z;5kX;rD&gH?aAr0FWMu=_HFWq8aHkvOXXEaNMztjv>*iTF=>W^nlWZLr z?H%F*1H)Xe4+mo!h=N58V{kvzyTfsQMwE@c(CN1lDq9z$x4@Mds?1;st zP#FR$Eq28^vjmknMMF!eMeAB<$BqnXJt-U1+0sV%sB8$3S>S~pY*Z8kLk>n1NDl_h zyb#!f0&ZopR*Va>srQ)>OcETF!5Ewg3TKE_mu9ONyYm8QDN;1Ljx3PO!X$=clL2TZ z#L1Lmj*lwKEj*5|HDDK}j5e1I>D7{r3Qzo`Y!K&Wpv!Za>6{J5Psv*TnUGinZ=>Cq zNs0k!LX%VB!L`bA$kgqDmqiY&4q`}FRK5J{i_fMiwk0aIE$&+$OZh_ye<)conJSsQ zpueXnF@@+Ad-=<)FScG9n?E7#8dz@pz;n|h4IGtr9FzFdDPBzQ;ysG`2HmZ>*b0cF zNAm?m74>KU2e{&_PB~i>&enxpN$0i;16gXvrR_sW`*4aKme}Ey>c&*{=0x@8g@`mZ zks1pn#zNAWS*d!nRQ*({{HY6l*%v1ky>E+G+TZ&6QdBxLDQ)AE_Nf#*C9zX0<*t__ zFGl9uQ=9fBHtkE6KbHI6(Z$BMwM%_(v5VilGA?ZymL{G|O`J(goVnO4*&``7BC(N` zn)R;)E(cOI9f_KbrCrII!BoZIh5k(EQsd60$}79Cv@Rc%+PS2CG{ufe?C46h`<0=~ zLkmsG>aD5rtrz<4x;zW zk>H7>$?r+&) zx%?@ao%#(Mxl=KChaJ6BzV$b3_{-nfoDV2%dhyy`Y)jdbNlFV z>BN)CqrrsZ9R%(p zu+!|dX_t-dUbA-jae6?bzezO#ezS=lXwl!?Py_gT3=M7G)9Ij>_slxPZCb!XnR5qB zL&k*u5Wcnlvk7hYgM)HSgIPxm=G@+(Lri|Hsz#&wHB1g2n$(XdSukvKj`sROM)J6w z7}eTUhS6Fkvkpz>tc%ERQ27GWO#eqrGi4}`Kp$>-d1?H}TIL}!GBYM~_O-0TA%7fb z8V&yde$T-#0>6XsYlNQ_3?SU-Ndg1t8(;vTsje2~Jbmca%1eIT1Ui05@|CMt_L*Z= zW(@mB)+WznS#wR{kl;5G11M{#W+-x|H4GpF@K>n4*`rMxJH{>}F?QgYXY5#pg?%s( z;V}R$w4&DbV?h^!ZUm?tg+T;|0Dvhlj5z8(foSp~#c?-+1*o?LqQMU%h7&A|BJd+P zf&jb&lz@w&5FP`p#Y9xEa12W)5THqw)v>~H%$`6H1R&x^nWr_ipc!+$!HQqi1Et8V z*a3&bf|^qDA5B&qOO+f$HEA{t(EoQux6BjuHkv9Q%~ex5>TS7S+U8H%$5QN=#E#)6 zP4@D(guQK1pS15!vHK-<|4L=eD>avE=FcW8Jt>DrVm+VQDEkEcsh%ntq3@KB-eJe? zu>0>gJYU{b^a60J()xeX3!sv>zx7PQ(f>=fpD2U@PzZ}KhtI9=-9W!acb71)Gl0Fh zp}U5fir-IdI>9ZtaCVd?HF=A9xf;>8*u z^Nxc?yo|sK0xJouBCy8X+n{}CLwj$n_WC-yXTAPA1=P{@k`?lVr4v*BhG0c^P1uiS984xrl)3l(KxlV?z?5TCBfaL%*P zHoW$x>ZV|hKn)Oz`W)|pp3|!%R!0|v@7k{~<3o>iUdqk_+#|K9m3|{<0CNQl2~1J+&J>nWBWLpK zi097WYgJn3LchVcDL1d+exuJ*uyjko(rpDxcN8c!Wuyd4p!J4>&;^ivak;yR7-vRDwch!qt2gQ!gID+2J@kD zlb8JTnrl=9BRA$Xhb-(&G*`}5?9hT|uc(LKMX&Ckb8_WdXn#@Ki>Dp7fzM7e{2Ky) zLSQd}4Foa(>EYiI*aapmT16y*2^)kj)C`=cMLAC&ItTL7c-aJca7gl-)v>8EPdHs| zq!#}AZCtVP^%tVZl@vBoiOi0m2;4@X35@nT@LPi4_f@Z4 z(Sfef?*0Hb3N9ybWmyg@o*wkF2`CV7rw0x#xF3{_Lh$LpX%QSg40!A&WlJO$n+?F# z^jsvaxWoe8Bj`UvI<^6L3;cB8S>w?$CNtPtz)uG*9Qh!Up<0ZxF)jo{Jd!tq=S~E7 z62ZykHVHETUsNbG2aV1t*JsK{K=m$>#(n+faQ`jHsC@NM`NGe9j#T(imL!fuT@0{Kna;Q>g zsh4zaY6xt5{KBUxGV?SE+9@7PR#4}Nk5*UNDOBf|l&zo)M^XIahZ{?U(Wa;H~JL{F=_@I^j!>8k5a zjx{Y!id(WIIK{_9rvfmRv}aMU;)u;BgIFwv&-2PL2WMI?Q@n~q`qJ)4;-ftkgwUF} zAe@0l0-j`L>r4y+a{>^d5XNvFnIS@|BY_M8y69yKk*5HWHQ5dX&hknoExrO8HL@ce zJ&~>wG0H_Q&0!-mELk6%odvh0Y?y}2apdMHy3|eZ{rzmA1=mDc9bFYwyy*>;0*oBZ;0PN!NI)bo@f^ zuN=b)ji-^Oe$M-NB24zlNS#FSo8Z*R434S6nR%8x|@h z=k}HL8{VjWwRWYxB~`y8QNJULG;K;XbtIZPR$AIpE&CHK`&a6kQ+3-Cb=!WmfADXQ z%ug&fEv>&@vrqb(?^j;`>wOD@OU^6XZZ{m1CQjU|psE`0QEXND7Zh7o{s6(J>#53h z_f3?|@p(T@u{G!%K6d$7s%BTBW|!37pR5^3RSbZ4xOT%U-@N?I|EO(y(Q;9HaqMnY z?QQoHiJb?N?kA)tCzDnDMPp`>IuhmJSiZ7-*_bRpnkqi}Ve!#Vb(DL{Jw4^9P#o=2 zW9OCD_c`fnM{k{29+4&{rTsZR?%Z&Xg|k0CmumMW+Pz8ZV9Gcs83$p#U$I@brJTDG z&RvUVlg`eRz4L+rgztsZ-#LA$e7yq{R2va5~fOoBGkzE=&kXj zYb;edmM9&=YFNhkv4s;$k>&Q~7HQw;?c)LI$x!n6AKo`o zPA`qUAEUv$eaAR>$KH9zIPiOMHz>2`w~jdJAJaox=3lk|cD0cn(lb|^HUoamK@XYq z*Gd`0E4w^HRhsJtdWh9uH`XJ*ksh+?uWzaY{Dzetvg>cy2)>mbD%0QCcAya|mMiEX zr+(SVAYP4BZ<^>46LZttRScQ;T=WR5f3IpI;J5Vjh>f{rK!$GZrbq0|tvz*szh6a< zlrryES0fJP<;?qgEP($U+5-e~09hJ3__>WhJAq{cIti>YkLYzjZ)_f+b-yU0hl`EB zur?vSgC1@-{$i&N@eTvf`GD3z%MbK8(aG41toM;;i%wAj zC!kij)YA&lX<}lCI6kspE88^(S=qVTvQTaW{|aiwP5>B6Q+BE3yN}PeUmCt^Eq!6; zJ2RIKB(1d&Fp{pZ1>ja% z5aVzY&PQW`fZ#?Bh})Sspig3!M*wFHN+3d07Qn0AcsLY5BPe`wF3yXAz?&3_no@+T z92+BWz)wV3{d4Mb4Wrxpd8tF!_E{^V8=^mF9l8+xc`K_Mpg*f_({26YB(2pQpcOLs za!V6(J~anx1e^=-aZ&=(641sKzK?*c2uilBkDr?bQ&GUEEdgT*1vGBuN?wg&^<-|B zMbAly7$B=I*A%Q*BicyVn%q#Z4Y^p(2FsG*wl%uL7&s;Fts<8chID%f8Ig$Fqi>U1 zHQHOTnagtD7I+Uafkk2ulY0Q!m2E)mOVJaF%mfIk1Q5Hk4V0h+PdW}r*+L)j=enjmh|NS3P+drb(KB7Dy zQ;&Z{wSPo)k^jD5JE|^?&ezY2^Uar^kg7K=gqN5%V^^4jr$?&pNjQ2Xs`v}$8#H}@ Mz7IF|@F2YW4{clkj{pDw literal 0 HcmV?d00001 diff --git a/evaluator/analytics/__init__.py b/evaluator/analytics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evaluator/analytics/__pycache__/__init__.cpython-313.pyc b/evaluator/analytics/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b08251813579198e84839c327946cfb5a5b22800 GIT binary patch literal 216 zcmey&%ge<81gTH7vq1D?5CH>>P{wB#AY&>+I)f&o-%5reCLr%KNa|L(erR!OQL%n< zQD$*TW@27`Mt*TdW_DtEqP|Ohd0tL_VoI@oVtQ&`NqkySVs2`Aeo=OOYFT1VX<|u! z5gr*NHF=3Sl_iUxw9C=_{hx2$XLV- GWB~x+{6As< literal 0 HcmV?d00001 diff --git a/evaluator/analytics/__pycache__/vloop.cpython-313.pyc b/evaluator/analytics/__pycache__/vloop.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..13b3e010b7c88c545328127806eb6cf979189584 GIT binary patch literal 4319 zcmb_fOKcm*8J;DV5A8~%vY&|T>Qv6QqXG@B862@UyDfywm zC^`U#vortv^Z392|G#85O|x@KH}Z}%~6Cd1Ledhx+O?2mi=b7_J%+O3()i4_xCyIR4*e?*Jpvb0@ zg^hp)V?Wjj&go=IHceHVz*#kyQ;4F$5Y7>O3ZIj;i5y#F+)&7L%>og@M%Uf%7~M99 z!yem36x(eS$DuNb?O`XdJ?sErD&ADCnFhUAY@5*}XfdK0LT^LX^hw>ARIkb^z%K{py5>7RG#2`PJM+7nzqx7c>9NG#syMhL4qD<+ zg&%qh(-^ksKPwI7{T)8@#IrFlh2RzH= zjC?l+*H=h^FV{*EycXuyG5o)34!bYdJkj$C@q*g0*A`Sm)eKVxK@^CVnr@CzqQx0B`SHTGs+n*^Kraq;Ruz!Oq zta|~CFf1Doigw#87Yd4&4g2jD!ZwoDNt(zdvYX27p(1X*XcFocghnPpO<^}dqSZAu z11Z`M-?(0CF>M>&7~8D@=}ezW*EAD)r&O)+N2to^Aq2MrA8oDrx|V!h%Mz|iol8>Z z!rPYAQE#s3h|)}3oxP}`3AnAO@<{=oxZ>)cybD7@?! zZwt4CxuMVfyOza3`Ap5p>sbYkg;2xE{|q1Ry599Z>F5qrz#ZHU$g%*h2`=eQxTu$F ze3E}=1jcxjaB9V+<)Xcr>tAz3B z2tFtX&eqpiV^#;+2Rs`Wh(4`mfly8=Sh+_1ccLhhERoebqm{6|{TN7TNx?)JFRJ;h z0Hl@?Mv8gtcx(iZfrd$tq?n=HmWCW9Y5>cC_d-NCz4{+07sgh>UQZ@{8b2URa*xNMG3x3- zn7th!Ja;0bw>8j512M3eF?XD*c&a2DJZ%i#KNMlH-y<}Q?s_1+cGbe`1UKl;abXwf zhGF`E@i41|dD|7~u{qP;4ljhXb8rY01;td^YsdD&+2HmJx>t(bxyozgJNiRRXC`yFdAn_14=~@aXLH?Hjjl%pd#J z%}U4ceR-+l^)G`*KkfdO1+$+CQSQ#}C>Q2g#3J-$wp-R_fafb_*hgNa9RUZ4B~{Dn zCDKDXJuJpUFs@nN_SJ<5&|YNF)Qa&>0QyI3hiTq;0uqDaK`M6*s&C6^Z98h|SQxRI zd&{RDh^@=PtzQSEx#PFg@*5AtE%nb6bMe_L^O5;Im6yYnz`pVu%f7%b-h2POxl5LB zSLJ&{mTzb|C_VC{fV3blgmmfz)Rmf69HJ^zeKCJ+|k&6Um-VLwVb zP3X-LKb_+>XA1DltnGj@6T;Fk&A>H`QQNcge~j1(YxwXVnbNbxyfR8o!8mn6gM#sq wo8!2zQ2RHi=_|Bt&Fkl4Yunu1@LIE*8(3@J#s&X%a0mAe_s=67cZjL`FIC&P;s5{u literal 0 HcmV?d00001 diff --git a/evaluator/analytics/vloop.py b/evaluator/analytics/vloop.py new file mode 100644 index 0000000..3964f8a --- /dev/null +++ b/evaluator/analytics/vloop.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import re +from typing import Any + + +def _normalize(text: Any) -> str: + """Same deterministic spirit as Agent Framework VLOOP: lower + strip. + + We also collapse whitespace because offline telemetry can contain line breaks, + repeated spaces, or formatting differences from Langfuse observations. + """ + if text is None: + return "" + return re.sub(r"\s+", " ", str(text).lower()).strip() + + +def _message_role(message: Any) -> str: + if isinstance(message, dict): + return str(message.get("role") or "").lower() + return str(getattr(message, "role", "") or "").lower() + + +def _message_content(message: Any) -> str: + if isinstance(message, dict): + return str(message.get("content") or "") + return str(getattr(message, "content", "") or "") + + +def user_texts_from_record(record_or_raw: Any) -> list[str]: + """Extract user/human texts from ConversationRecord or its JSON dict.""" + if isinstance(record_or_raw, dict): + messages = record_or_raw.get("messages") or [] + input_text = record_or_raw.get("input_text") or "" + else: + messages = getattr(record_or_raw, "messages", []) or [] + input_text = getattr(record_or_raw, "input_text", "") or "" + + out: list[str] = [] + for message in messages: + role = _message_role(message) + if role in {"user", "human", "cliente", "customer"}: + text = _normalize(_message_content(message)) + if text: + out.append(text) + + # Ensure the canonical current user input participates even if messages were + # reconstructed only from observations and missed the trace-level input. + canonical = _normalize(input_text) + if canonical and canonical not in out: + out.append(canonical) + return out + + +def detect_vloop(record_or_raw: Any, history_window: int = 6, min_previous_repetitions: int = 2) -> bool: + """Offline equivalent of Agent Framework VLOOP. + + Framework logic: + normalized = lower(text).strip() + history = lower(history_texts)[-6:] + repeated = history.count(normalized) >= 2 + + Offline telemetry does not provide the exact guardrail context, so we rebuild + it from user messages. The current user text is the last user message. The + previous history is the prior messages in the same reconstructed trace/session. + """ + texts = user_texts_from_record(record_or_raw) + if not texts: + return False + + current = texts[-1] + if not current: + return False + + history = texts[:-1][-history_window:] + if history.count(current) >= min_previous_repetitions: + return True + + # Defensive fallback: if the reconstructed messages do not preserve a clear + # current turn, flag any user utterance repeated 3+ times in the recent window. + recent = texts[-(history_window + 1):] + return any(recent.count(t) >= (min_previous_repetitions + 1) for t in set(recent) if t) + + +def vloop_flag(record_or_raw: Any) -> int: + return 1 if detect_vloop(record_or_raw) else 0 diff --git a/evaluator/api/__pycache__/main.cpython-313.pyc b/evaluator/api/__pycache__/main.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..448258a47598d42da66251f4e85c2cf4023efcd2 GIT binary patch literal 3466 zcmc&0OHdohby}kjAbzp|Ew-~63n-8Q2`A;s8iZ1=WT_au*kRUW%TP6LW?E?QXhz%9 zG6+nSvwO&%Qd^aBl~0@V-g3@4ha9EofHPUD^4W`TxJVAZ<-Hk6E7)ZZxumP6=k@#b z`*gq8r+s}r5{y6ob9(3FE=l?qE?OfJ0Oa(yM3P=es-%(?X_;`MARJl=Ez4Z?VRM%2r$Rs3WrxwWQ~1XVXi=tW%d4QOJsF_ z^v*W3g|ftHiS0W4iPF^IaNvyyQ&?<2mMt=BiK5F5_-i=cX%6YCA>8Et*g0F3tYj*|jZ)8g7=NJzAk< zkH!Lz9R59&4xX}^Em70Xl&fAeu$m9T-f{TcUYMRsuSVW|DL&U~{o}kJ5c`oVuh_2M zJcWRfq%eG+qHn7Jj7bXkx6!7fr?<%(_wY!${#H0HX2k z7)-ChsjsGgTKnaCefn2N(_8iFt(snJOq)j&X6;?GK4HE+Mb|&MHbK51W8vRNCdhu$ z_d0t3>NL|h%$B*c&6(@!nAg_$ALH;5xz*HiaUA$1*rerj0+R0gthLNYhrDQDnipkH z;Y7GDs#Ka$=tWwKmtO-)0zNkd6Gr3f#Ka#e^~B;);(k4GzxH6gk93rvm~f){L5E*0G~J?*Zcxx3c+B9?l2gq43u&9AL#y;pfXQ7q99>k) zOeuwrS7* z_($cvLqWGJcCh7egYjA2u`F74*?bFF*fkANoWD6YH*qM88BuI<&fhHWDXwFghB9Ut z#zL7IhG}olzlR!uiRw?bxl^%?`7y@WVJ;iklf%*mUZ7pz*Uj&91gN=k{(hlgnKo1I z-TPpcy0dh4xngWHW!Dr%<$M$bvsj4;XA8dT6G z1JCTZjy(k=I8l`nTnjpCayna-HD%{Vr>MO2b0(el*ZV*YW? z4;kc*Dn-r;i>cr;k00!1urhQOv{b$Wo6Ayk7muysWOTFs@ zerr6_3$QzVip60QfN%H24JZ@Act~1Za0BazKh{wC+(@-wduk{XQsvvBaS1zQI=pKB z8%hzhfC%H<;a<1kRA+dF0T&M*7n<%E%!|PR6Oh4ln4|Tc>ImhJ?=-=x8R zMEhz(*?M&Dx%_t${!Q}N$TKDw^Ng;j1r;j0hgS6?N4ZnQ$JXs?%ZvekOfGTE@WqrVB aIU*xAgent Framework Evaluator

Agent Framework Evaluator

Offline LLM-as-a-Judge with Agent Framework telemetry.

RunAgentSourceStatusTotalProcessedFailedCreated
''' + +@app.exception_handler(Exception) +async def debug_exception_handler(request: Request, exc: Exception): + return JSONResponse( + status_code=500, + content={ + "error": str(exc), + "traceback": traceback.format_exc(), + }, + ) diff --git a/evaluator/cli.py b/evaluator/cli.py new file mode 100644 index 0000000..22a017f --- /dev/null +++ b/evaluator/cli.py @@ -0,0 +1,80 @@ +from __future__ import annotations +import asyncio +from datetime import datetime, timedelta +import typer +from rich import print +from rich.progress import Progress, SpinnerColumn, BarColumn, TextColumn, TimeElapsedColumn +from evaluator.config.agents import load_agents +from evaluator.engine import EvaluationEngine +from evaluator.persistence.repository import EvaluationRepository +from evaluator.config.settings import settings + +app = typer.Typer(help='Agent Framework TIM-style LLM Judge Evaluator') + + +def _run_progress(coro_factory): + async def runner(): + state={'run_id': None} + with Progress(SpinnerColumn(), TextColumn('[bold blue]{task.fields[stage]}'), BarColumn(), TextColumn('{task.completed}/{task.total}'), TextColumn('{task.percentage:>3.0f}%'), TimeElapsedColumn()) as progress: + task=progress.add_task('evaluation', total=1, stage='starting') + async def cb(event): + state['run_id'] = event.get('run_id') or state['run_id'] + stage = event.get('stage','') + msg = event.get('message','') + if state['run_id']: + snap = await EvaluationRepository(auto_init_schema=False).aget_run_progress(state['run_id'], event_limit=1) + total=int(snap.get('total_items') or 0) or 1 + done=int(snap.get('done_items') or 0) + progress.update(task,total=total,completed=done,stage=f'{stage}: {msg}'[:120]) + result = await coro_factory(cb) + progress.update(task, completed=1, total=1, stage='finished') + return result + return asyncio.run(runner()) + +@app.command("reset-db") +def reset_db(): + repo = EvaluationRepository(auto_init_schema=False) + repo.store.drop_schema() + repo.store._init_schema() + print({"status": "OK", "message": "Evaluator schema dropped and recreated successfully."}) + +@app.command('init-db') +def init_db(): + EvaluationRepository(auto_init_schema=True) + print({'status':'OK','message':'schema checked/created'}) + +@app.command('show-config') +def show_config(): + print({'env_path': str(settings.project_root / '.env'), 'adb_dsn': settings.ADB_DSN, 'wallet': settings.ADB_WALLET_LOCATION, 'langfuse': settings.enable_langfuse, 'publish_langfuse_scores': settings.publish_langfuse_scores, 'llm_provider': settings.llm_provider, 'llm_profile': settings.llm_profile, 'oci_genai_base_url': settings.OCI_GENAI_BASE_URL, 'oci_genai_model': settings.OCI_GENAI_MODEL, 'oci_genai_api_key_configured': bool(settings.OCI_GENAI_API_KEY), 'agents_config': settings.agents_config_path}) + +@app.command('run') +def run(period_start: datetime, period_end: datetime, source: str='langfuse', limit: int|None=None, show_progress: bool=True): + if show_progress: + result = _run_progress(lambda cb: EvaluationEngine(progress_callback=cb).run(period_start, period_end, source, limit)) + else: + result = asyncio.run(EvaluationEngine().run(period_start, period_end, source, limit)) + print(result) + +@app.command('run-agents') +def run_agents(source: str='langfuse', agent_id: str|None=None, limit: int|None=None): + async def main(): + results=[] + now=datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + for agent in load_agents(): + if agent_id and agent.agent_id != agent_id: continue + start = now - timedelta(days=agent.days_back) + engine=EvaluationEngine() + results.append(await engine.run_agent(agent, start, now, source=source, limit=limit)) + return results + print(asyncio.run(main())) + +@app.command('progress') +def progress(run_id: str, events: int=20): + print(asyncio.run(EvaluationRepository(auto_init_schema=False).aget_run_progress(run_id, event_limit=events))) + +@app.command('runs') +def runs(limit: int=20): + print(asyncio.run(EvaluationRepository(auto_init_schema=False).alist_runs(limit))) + +if __name__ == '__main__': + app() diff --git a/evaluator/collectors/.DS_Store b/evaluator/collectors/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..eaa3227e5a743beaabb37cf0e3621d5b32b125c9 GIT binary patch literal 6148 zcmeHKK~4iP478z0iz=ZWIB<-_3o7w|v|3)!8<&M{QF~~$d+DLKya4eXJ^;Vr4UA(` zSh4iL0U=~d@i>W{ct_eKCL*407h|Fk5fx~HEJlyWaPQQSIS+uG*Qn`^YFg1fRXq#+ z#UaUkh-@orX@}fD@~_s*@vLq(t){)a`QCHWl;wQg%;8a7K0iI2zrIWkc`x?fC)?Aj zvxDC4>T?#4nv82_z!`7`oB?Ox=M3P^7Ab~`uAKpAz!}&xAm>9s6O4vQF&`aJxdZ^p zGdc@wsU;*P7)HaSh#3fLC{RP$N(|O;j0f|JhDlMwiLLlx+xfkC;k-N6580hKD!O(C zoPjO_$J!mq{eO*LrnkuNhIr2za0dPv13V~a~R;Xbw_UgHuk}-7}XRW5% zg$X62)2tP(of}gOL!MIrwQiisnVcfRM#ZX=77SgJa=M(gc^Qgns@j~9aRhVPij>U# z9vELBGlViCp{&SIPGs8%dYL8bIAeFO++r3x}<8VDH)l0 zB`?Q0iYzEzwdFDJynx0VN^aIb(t`Da9Vr8}xg$eW!&GHWpVy6f^`yKc@95K)G+Qe> zKx>*Q&AReS%4s=Qkb&CQtgPc?+)FYZMekXB&GJi*z>+0AK}$U@MgSx>BGCMh+=&Ul z+E z#jrd%HLscmN+94zn{d0&rP=W$rB@K@0o&LI%{|ciYOM9*^!e%Av3NNazqaky+pllG z*`3yd5r<{Kay(_?3Yy-$-~9Vdf)0lL+x21v~&h zbYIPYVjKRSpuIqL5Y6+=@5|LaOEhoFJ5g%~Ko&n2L-VD46V*F&*02?sCvZcXd*l<@ z-RL{;H4u^8#ii?<5g}k9yvXn2c2`_eAiyo;uCDi0Y5XaFlngRMV}umEge?Aa@pp@# zq&U~cWRP^~>OSBm1I4jT1nF%g{gZUHk@Vgg>3y}*rvjpX5A%&>55pZrQsF2Wl0kaV zxz?yPX;H0Ni=`qH;q}wHP#O)_{!)#prpLS_kd9W@n_U^xF?TNA;zFsY7<8qqY6`q2 z6`g2xan^fR*W*kox|6v3#JWAa%R?Yz8A9s7Q`1}AePVqZNkyVnyAcEKPHj^SR;Z1n zx4L^$5trhI`&v?-4YktY?w`7x3RlnLhx3HR1~g?cjO{oioT*=>Hlo>=wrQClh^%$wXz?JJFw*?C+BjlN)jn9QIbop|R7g!xQ6&`zwk}PJWFJaY+8zVahTfg1|dm&pHQ1D4=R@gv6k5Pd_Cb+)EA{LM7GR{M<=M>Cvnh9l#70kfA zso-sZN)%IuVp4vn=z@$xc44hB1AtpAKwRr0L~q=vW)KNx#4N^S$Xh99q7v_6vihAS z7d#MIwmJSY$2eIYich`(1()TSFXUw{d7Y(L>Cxw)xz1W_&rXZU8%WO0)Pem_d@036 zdeGYSq+IP*2tE6AdPdF}2YafUo(58e01f8vl5p%s-?dY}ANX*fH1JA!|0`EpuQa~b z`flruzOU{>#364)yP3=I|9?{tX9Z-i0GL2ReF2(U#Y<2R0fMPCBUd;fv&9uH7FNKr$U--H)| z0;XpL^UgB?VZT#K0p!cf!;?hGSwU4>ESQq16ius6FPPNFBzIhp* zJTQvmhzE;+O;o{`kdmcPJCOKhyE3Gh!)Uv`r&0sbr|c26-$aJRqc|1^vOv&qe9o}7 zKwrkK2}I^HX3^r&TWB|q!KapBXWp1I)(sZi$(WN6$*M!B;%M|cNaheWz0eq&p>bo* zqrF{?QcT-lQ~p;x`TRYzwxjgdD;me1wb*!}SDQ(%iM)-Z9HJ`6F@}Pk8(wOyGm$96sl@Qp951*@db@vE_xbNpY24wswz&h=e- z`pUPjrEfmF+PwMV==srG!mgXme^~nX_0q)TEje3K7FXm`4+wdk8DgQEVKeL+#-JQ_ zdDdXR#7UF=%;{iqm_3S1W6VFcbd-f-zv){Mj+MTfUfD8pE_kP1D0k*Qn7Wz$;P^^suB0ulw4XXxw;E}= z*l@n#R>$)Zh+xM0sYM7OYxF4Uq8vGO>Nmfvj z@_M$AL%3d&P8H-_WhE?0vnn;roT@3BE=d%NV~Vw+<1H}>pMIvAfwX2)^#nu{Lz1o& ziUqUdRd@o54MuhY_y3sui)B2$%okq9^BVKHcZ;XR!AGoQyaDBzEtmxg@=(l*_Q9;y z>RZ02;)*&odQQOel450Q#cJ27(gR0Qa)uq0oG{k4jsOo;4DX7TQF?UyR~R@4{4C-8 zPGN_-huCSe>RgZelZ^d4hJ7S@0;Q{-)$;Fm^mvfY!vYke#MlPS8pklqUD9=zY`shN aeM*KuC4+ZK&jY@TX(>Im>vICVP36Dq;#lYa literal 0 HcmV?d00001 diff --git a/evaluator/collectors/__pycache__/base.cpython-313.pyc b/evaluator/collectors/__pycache__/base.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..423b8fd88b29b70536f6d9c3825a2e38769537c2 GIT binary patch literal 1003 zcmaJ=&ubGw6n-rMXzVh@kP5Kn^aR+A>TwlXTO`PJJ_Ju_%aw z|3Sfj!T(4GiU&`#{0cZiLB(!r|qH%8v!=$(dIo zRoG37%7qP!7a2ct;0*QvNdia{g27&xL`f+L)?pA1$jMoEaJK8?$E^=9pcl2W8b z;YG=1w$rH|lT*~$PXJ^*@4yz++(mtO67)z++Fv1#Y^a6o+4&=Znd&laI2v1>dLr-I z1>P6AVO(Zh3vDBr4Kt(t6*a%CslV2)$$E;JHW|;=fzk)rAwS{um+B;!ig$IrsL?|{ z6ekJ~nKTav;K6w<49~Dy|v}(p56GT5Zlvm+0oR#^9%R@mlQH8>2K|Y z|42iEJ`&S*JFeoQgXrbMrE7HCq0=QT%&wdAe(hbYt#W~P{BAYN&2+<-P?!muL@(iYo0M%Wo)RrBkA!xV{b-WP9sZ<^)l2(X1T}}W9UW36=&=!^OG@I z%D6F@bw);Lm*Dx2-2=Gn&D_lRKJ-0jKEt;fH7W@BVs`a<8ctdi+pYy6HF|xEYdw{suBT!vPm3m>;RENo9~QI+mjNN< a95%nh);VmP!{c8eC6az=T!DX23j6`jnGN9p literal 0 HcmV?d00001 diff --git a/evaluator/collectors/__pycache__/langfuse.cpython-313.pyc b/evaluator/collectors/__pycache__/langfuse.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbbe06f5fa290d75a59b6a19d020e53587d00a06 GIT binary patch literal 19001 zcmdUX3ve4pme>sbKLQ{D5+D9RijoM4A|+WLQeRUNC6Sa!iO}dr`EVcsl8`}wngJz| z_FiP~NU%&pne*OBr*L+w~Vxr)A?|-a4`#W1G>I?jkAFULj zAO0F5uTTLhKzCDJG)wz1uIg5GsaRE)npKl;b+@KV%W6qn)2(B50Hf{JcNtg%Nz-*3 zyG*Q!#P!|gE(>evDq%~=x1rnGWn*oN;2Z&?mpZFq?Ez=NL}I0Z@_?Dd$^sPu3yC=b zu0RQiIRoy1Ct$_+S(+^;U$)#=MZg=d6DXIPs;2^_UMf&_#;{o}^^tWCYy6d8;fJ3V z&7pWakqjlHi8$w1iH2|}8A(P*BccJr$wWK?KV2_ae0D!;yG0nw;p1aETaT9;Br+8Fg*myt$*K76$MWRq)5)>?J3zP5aV%-fr^&J9OK#4!vCk~S z=ii%i(PHfhr~>NEDhZ&0@@sHFb;-$Sd?+!gTf2V!w)N{p`m|rimH>>XXCq^=&|ri` z$`sY($wAQ+;^a9IjZrQ!lwe0gNx-5CCpm1CPrTT|a4DMmac>enHsndK(Q%aRJmt5I- zdFN~H8&%h<1oNtld41ZvUNCRWn75_P+iths2?*v+UeoyykWs~n<4+*x6{=q$9ER%8 z4KhW^!`+C*;R!>|da41ZJ-6VcbSVtb^3?963rqN*Oq?#LfnK_X5Y)iX4Wt&osz)@8 zMv@^|Qz5@vG)1{+oJ)q{gAq{^jt(Y8^>8E!D@L6dI4f!plEbF^e5?~*qA@5JW^3RZ z)3{v_T%zX8gn(s`!WuH z+Tl-a;x!v)SqRt)KrskE<-ygV$91+5BDl^R)WK2;y3EkX2V0;sn>5>DfN}T^+pvS2 z$_K+}rhL>ebrZTt-K3x&IwYhBCu*g4VP7o4p(x8GgCvJt2^Cesk6R7FCF;{sS4x{T z+ozq=?99d)mUplDrFl)(;r^pT542R7dtOIb>{7dxX@-K>Zo(htpeAfL63B4jL@h%K zWTK!C1=CuztH>^g{IF*%0XDXIm>8AlwD z##Q~e_Q}JjWNdLYqn2wVGvLA*=!1D!{4i>{jE_3Jh9dJ5Q1!uFE`At=)EBk2Oon`- zt?y7&Kn?9|fp%(6KD`|PNo$~Y1+Av~kO3@u7>(R6`Tazn)k4Ys^)EeE-UapOiuOe>W0l_x zNV`R;$#0n4Dea$0-=Q4OZRs70#M?X9amk5T#5b6TCxNN^#zGT>$NQ2AA8a4dxGx#G zm~3Y2pr0ab^6NznCW!i60Z~6HZP083LTID$vGJs+ON=M+Z5m9Bj>SM6NW`JUNFq8I z;aFVNED#sP`XTU}Sr3*}b4gZIN8?FRI|MCAikg8$A|`5Lun&p$V6M7gG7%*8>q2AD z{IIAwn~26)7%qxMe&E;V+A$VllbooDCL^OP&Yq}#HZs9Blh`OX%mKRsHc1}BE7?tu zK!SRRa4UfkT%zvlEiZfzxZ|9@bZX^mj*PxKt*_47oGFWxvf{PzH^RJUZARae);G=B z%BD8IG|3x0Qm!+juSx4`vX#{-?Wayp3i#=YnSBDV(tW?OIa0~2z5JTxs%84v$M*UM zdaB$zZ=meXD-)L|rppCeEpMn@faeQGe`;^mR`$ow&$(95YTq{AG~QbAW2@lWI;HuQ z!!@-jTUqJA8zsRrnPDz^nd7P8Hj#cWe4{{+)m0n4xGUb{J0UtGngl;w%61nVcy$+CdD zP1OQ>=@a!CWSA%F`vjPj2AI@VU{dOW#YzY{8-~){cNyW&w-kjy#)=Yx)&74kKDb^b9C$O2$@9#!<#y zTfi1!(A}@n}|gurmIG|k{ql!(E_|x zIg^PRtcu+M!$BZL9hVqqLEbdTvbtZL&t=^J=Qj}!&ceP98vr(IjKwoio-4CYVU`o5 zQK?EeIwVGOk{t3~Q3qQJaC5&w;@*~UWGFNqO9n}Uadoq8801@}h7E>og)|PAa*mNp zQZKs~K{3D|_q@!>r<_yl^u}qHx7Q2$dLr|Fxrnb{dG7LaFMThi`MmS+J^PB;s(3RN2X#=x?+z|u`hMt^XS+;=jz#GcSrb>1N`B^v@^^*BE01yfBqtbU)pMC zw1RC_#@3X!H3_z6=*QVO>BrrR{HZ~{C!DrNc-s(fIL{}~L%5(nS^wIUnru^9i|tC+ z<*s|sjeUY^_s5o}AE;q>x^F8>nZ7hvPIn0A`i!|TZEh6IYcB2keCX^wOYO{=yRAGE zYO1 zq3SWEF;-{IwkxfdTca$1;BxNZQgt;~5 z;+b5iEYDl=BQ}S#V%(aiTkbq0lM)W=eMvqCbB&8$*WyB;CWwAai(OY!$ zlwMw;g>#*od-~yzAoUgM48?-thgAh=Rvl2Wnt<9%vDzA{h6wD7_jFQw62g`+I!$zu~urZEtl#)zOt zst4?c4juM|I9~*kC-4-nNOBx$ZW2HwOrSBBu*j90>i6credj}y(NKL$J!sL#5Np zkQDjNrUjZ=q$a{;o+i|k=Q4S%X`cqN7Co|%z)&%T7lCq-?(u6_Jnpb4DvHi%&SHQn zg`wzha4eJ@5lfX+HfKlxt+=hBo`PIBaxZ>1sp2|D zU4R*2d{4pe=`?cZ@!&LVYpniQ|U2z8?((2l}t~k7<2t1(&&@NlxSHf6ej{XZAnyvjPN0NuJ_M=&WJn}(#!3;u;jJ0U{O62toingU=sDck^<9DQpF zvT{)c0#p|O(XBFg3tx^}#Q$ zAEFG@0lH^~gfAJWo?pQ4pTV!Zj%uPvS?H^@#z3UVb1fLFA;3`!YuZPh0E|1;wbT&3 zRXs%etv!=NGHZoxj|&aNK%0?ZeIPv%z8#8jzR@TLcCg`QUm%k7b@a6FIn)t6)ZTMo zKNINKnPkT!WDk;4f=8G=hdTELyE{&`1UmNibsWQJ^H*UQ53s3~pe_OhHzJRT=42xO zp^ZgHqsgyO?EvZ%!$63O1jpH!s7Lt?G!TopEIQaCi3~TviUrE&&?so1 zL*vPjucn~Cla)%Ra)L$skxt(boXVhgld5e#Fz11Z53IR(V&g63bYGjV8iDXblU3bnXka z|0UA-H57}&)NpJJOP<4k#d35Aj>%G%s2h~JX@LEG3_KMgqCRKpV8;;_tmW7$j_tuO zgVdjBnC-<+%YvR9#3Wq~9&M_!`l;|OiSA$}A)=G$k>@`zFXXcR7!Yt8SgBu8m#9Zp z%I?nC8sYz3Sxwe{@u6Dfu+CGe66+%qRnstYNT}X0S6Mf+MyOnuE%#nKGd=h~uPHO7 zbdX=_$=I9H_NKQ+GRe=O^k=JA+&F#xG~c-6&hhEfeD#Tp>qP2c z*5=CCR-|n!W;nsNX0D>1U-|SMccz_5w=?|H$NA$Y`BS0HsiE|#A^zkrzhXqFi1PMm zwxZ_R=+)7e6R88)@~Uh7SNlQvRo<8>KQMdX?e3f1w+%v5JAB>gOYOhEeq-vur=Hp= z?VPJ_X60MXjH@~AYW~Dom96yMXuIAvbL`dKQ-;q2{nN=z?bdYd*4w(f4MJ^i#?#Bg z|Ipx+?(AJ_J6nEBciVDjMDX-Z>ENK`sm-|k@PGDrX5DW1e`m{I?f$FX*d)8_ zn)Rx6`uxZCRS)VYukUdwnSd^SCkx zl!yE8+z){oUTojmxs(3aZKlpr&Cm5Z3`_iw^7AcpXNB(PTh$of=IpFje_*9MJ-QEU zdv);fi*mZN%J7SddW^Tyoi&DEwC%x+ySvR$;zJYN>C=2@wqV>tL;8nRB&;E!kH++R zUFSCahmCaSTFZxi3&uAaAbk)Rx$>aE`F#{u8-TG%;wmV+Ainqs(2NR%85IL(z&6$5 zZ9A@EG{uD5fSL$Vupo$^g=TphM1fGN%^TX~ja%L-LtxD*ge=-xMd(@v4`_B6?ee9; z2&|R7H()DFl1)w9GwFD8qiFHdgiWy=3>=$;z^^52m;DEra0vsnj1o0EIQ1YaWM9JI z3I@pG*dJg(I0(8MNc=}KXTAX6;>stvP7!|nuK<+$eF%^P&D()~Z+NeJ`BiPV8*h90 zhJAN8O?!FIk&NR=%8)fXGUnQ}xpw9$@V3c1%2S=$GVir@SJ%DVvdGLhw`S9=6F=*J zr~gicu=(I$H)PiIrq}dNZN0Yr>h|gLf@2kLIm0t&ASAl@2PKsADE$qGa{Y_0J$Cv! zy~nEgC+(Xdx`6%+06^&PV}<^Z(#XL@wFz_%B(7nvOi%V`(2?S? zs$PBhP-zmWAr$4ek{lEsz0w!WFOBo^VK1hG_wS{uG!^%eIix>TS{q&FNz4f}c z_NXBG^Nkt|w|Mq8=ss}hF!Y!p<%0$-gbUOpB~VQ`WQ$m@mjN{$5vB1O#s z1|sbe2AN_8qUH4W3OuMdifj0lS_@WR7p}e~;Fs267dIgl!w+i=@E38fv6->SZ7868X>}Ecw-E(?4s3z4C0e3z8palIfJ)ZRSQU8|Mi>=J0Visj`cXq9NTqNh($5au&LnMO-rpltFQ_{r`JNW7 z#l^Umt)={A8OY)upm+EBrFJx@OiKl8l&rb+R0q8T_fzQl@25;r-r-%kT}!m8*j_G4 zAI10bJWnW|zf3QSwzyJm@e=L(-^EuY;|o+bYe4Gqz-o|*CQsN@-`awYT$J(1W#iu8 za^}5^w`ebGWXyruW)-xma*5tL*DI>;qIi5Vo&p*gSVr>F{TBGP{U6}_ZQ3vGRdw;I z{&n)4FuEnivA#(EN%h0URs9 zXAlg<;OS(_MZrZzI@8gTS0ECTXaN;DD?1nx?{d*_4z7}*k(mW{CFx2;T`107h_Ipt z!9`;TaD!GE)Zd!q1n8B!5tX^%SSC6ZwRk})xIjLQkk*1UblkBQeiMXf-`j=SGcYEP zh2o+%avqiKCb&Qq0XG6{+&_ohC(PeZ;X&s|kdsre;g-7>ETpAmI zOTb7ICsMG9sG8w!+UUaDNjq>9E{?Ig#Fiq8D#cRJyd#zv1iR0!W~IPt5GJ?{P$?dk z`I@mmYn(m${?NNa@5kPay%*G4Z2`KWqHvA&Sv2 z*0q<=Z`vUE3kL+m3FrUgDNy6+t3pv0j+qqu1_V9q+ZcTZg2EwZe~PL95`wijfIwu@ z0NO8n)Uajx-21EEUG;v`yG`#k^M^*#+eTjY@T2j}==ooco`2ahReH_yCmw+NhW&FW z2TK4gCB!~LR-89%n;Apn@2#eq02i`a)y?EL0MEAfP5m2b9Nn5YZwq`}-LDLkx&@ z374g)Why4jcLuHaeb-i zTH?NMlP>F23bvKt-&%sV4DIFQ{!SIB$-uuAlHf{@d)`bL9rI;W?aCWxub-W*5o%g9 zHM?%_;SH5=*@mz1zXcaD@!o2-GCX~TuMDSKyM?B~k6OELZTRa+{&fGnXF{3YfwzxB zdb(*408)qMT-DRDd+R&WuKg*?mlgHdTyrl7mf9@1)OKF(oEjD^HCc=O%E8MAr%ni# zYB~Npf~7%@pCa%Q%xS^m11~?9DP?|Sr`!M|4Z%f7mvN31z@IF@k7Z??nEueLbDL5Y z&;+(@0JDL?e#Lg#cF)~3+kbnN;BLQX*i(>p=Js~Mec+zq;FpeCaH@CtW*jg7Uh3eS z%RBwljjh+W&TM&gcc!vucJS@to5Qz;ew=`hJJo`#H{)W`E=C|m$(7sha{uANhZp!0 z{r~QJJUf~8K9}nL+~&%z**ImM?n>KNKhjYZo@w{X&!qNeEzT>wmwWG3t-s~?nd=?b zZI`g2{mvGls_UMmJ6m2oJv2Qml&_kiv-Z+!rmLomy)JF9n_2s@y&0V0D;A*I(8lk4 zR@m_D>udRQ&+|C^i*UPz4SV=~Cxoh#_bjKDNB#(rgALK?n_opa!7#fn?N~QE0vE4R zhA+!k&O9rWZ_JdprpsG}@@**-UKe;?sMwUL*p{x?2G<2r=KpB(%z3KQHDm8I3N>Rq zn-o0bsjjSNMaJV#d;EfDZK~^Yn|H3d?i2fkHxAC8ya)HX>=&~3vWHrAjWyK+BwOjv zc$?GSX2H8Y)qUSxeWxmI?MbQUHTo(OTz?}Z?iFls-(k(HYF-Vg@>*~Swz`aM^-SNq z7E)nB)USTi^SWntt5Ek;rmpQ4oAK>R`*!iWj|#rNjPE4ainHEYa)Y89YRh={-s<>S z=R2LZ!$0YjExjke)}5(*>Q-6C+nV;a-mVh7`?Gbcmd5*a3DnIS+_2=Te3{A(>BZ68^5hfsO-iZjmBB5Cy;vI2g`76rm`hn*}|`H7b^FBt%u~40q)e5R$Mb)HBPUY z84&FLjD20&zHT-u*jpb{>Qax)OHM$p!=IbEsNiUQtcKkCIZm@6_w4O@!Lc*rXiq!Z z@Aw7Bp~qSPg;8^Q=8crmj_v^UKrmU?>Wph`8ZKlU7F^phuKl-1lq9}sr{LO^adq8k zL`Y|4)>W5vufRC`w>ut`o9(7~%4{@!^L0c;tqaqf1$g5JUeAfOx(~MMARI&)PJT2H z0fB;kSOX%$E1<%KJH-%_9$Bh{+qX1Gq?4coTEdLGSl|!P%NofFE|U={v<0M4O|iea zQ0%WQ$gN_u=y`Z@P*xuuhOM7b7rW{P-ekEx`KXh}VoLu&9gb#(MNj_rG04Bq=hQr^ z28#6oAlx{84qimgWpR6#ZYfE!K`0!SJXv6O$1Jk%?uQuqmV(>>r@dUsHi$iWDfQnV z=;|W-TrmmbICO(UJ2G4Yz{@;yD;ximNohWHRKmScm-pIpSD(wc8fOMF6)owC7NKGT zTq$yR(Kh6$xYl~L_1dnhy9BuJd4exnM{b(F^c-g9Hjuy%4rje}Kj_Zg4w~ZrQSSo> z<*b@_Qg%nmMEqL8q8*0%;m;QIz@@N1fGCUP#QvcYOF?YX20lpr;A8ZR?|EMj{CyB& z!J7nspkyuqy=(9y+oT(Pr~99D7=8wBioFD=zAYu5Ow9$3Op<5>^dp(bzpYM!C>Fu$ za>$$T=3FlR42u@GZ>@NH4ndlH@OMgNUkr^OqAVApn%1i>i@O`(|(t>YRm51>*lrahEk4- zc|FDql&xvrh%pmoaz8R-%;(I3!aE0wF$ol709y0NjIpM3v=%n2>Uk|j0JyYvUQeP1 z%C&ahNTMdnQ9W-aQD{)rqY{ih9#WTR9a8tP({O%qz)K=f7>#bRiQu zfkTP_xD{ANhXF~&7^F5VOhv*2+jTw_j^85#jv}W)YyzFki!>yf2X> zNhPQ~NIgU2u-JflSm67)4uRkdK5n4G06$Bqk-+yT(G$Vb1pcL$=A8Rlg5{dgFOt&GMi1k(?RJ3(K>SW}w?hjFWDQ;u;Vu^8oTup;i-k9`ZKEH3(E30)b`J)=3h~r zpHZz4d8AR(UGyW1hTizds-a<9uAwU)?V)REj{dtnw2{{Rom&kqP>(DITKA}8KutS% N_qxBOFhU6Ce*stJwYLBO literal 0 HcmV?d00001 diff --git a/evaluator/collectors/__pycache__/mock.cpython-313.pyc b/evaluator/collectors/__pycache__/mock.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5dd451942813c5bd09553f86f54f8084fe35e0a6 GIT binary patch literal 1575 zcmaJ>-D@O85bvJpnf=J8rwe)@jxpk`lHH3C+l6_wr=JR{g4~xB6Gr)pIsARVT3i z6tC@l?-KF{8l&Y60X^#gctAoD(p9oT6%7zOtImp}oC3S6?g~?^z--l9@s+<)QI&u^ zOXf)ET_mA@gDpXRT&lWLX;x1WHmRN9StboPiOe*Z)h;)}Bz*z6u`ILOLg^y7EYnoP zMyjS`FPsmoi&(0zefqp_O=!(`M2u&QR(U!-n*+}Sa)T%u66J(cxuF;Op?i_+u|29- z$dtF^G%I#$O~(7S0`4u7b>{|dYkpR+ex|fO&A-qAz7A+03RUS`Z z$5QDH>5?;yU#B6>ysdHW&@l6 zbe_E7_GojuW2cRPw#j}J%gk^+nCwGM3r6u+EG!PD3rj<{p#5XJKkVMZqIHkN_Ta){ zE|d)VNw&*_9yht-c?%eY`N#}}Hb4Ckh>xLh z5qFH8khvKf)$)JZ0x}#1j5nEO8E#~W5lZ3BDr|Dow@iz)XD1JYN@OR{hmC^QpvlzJqOGqgr;g`zAS{QBUz@!6*UJfq`I4Jx=6lK}x#&= :start_at and CREATED_AT < :end_at + order by CREATED_AT desc + ) where rownum <= :max_rows + """, dict(start_at=period_start, end_at=period_end, max_rows=limit)) + sessions=cur.fetchall() + for session_id, agent_id, channel, ctx, meta, created_at in sessions: + if aliases and agent_id not in aliases: continue + cur.execute(f""" + select ROLE, CONTENT, METADATA_JSON, CREATED_AT, MESSAGE_ID + from {self.store.t('AGENT_MESSAGE')} + where SESSION_ID=:session_id order by CREATED_AT + """, dict(session_id=session_id)) + rows=cur.fetchall() + msgs=[] + for role, content, msg_meta, msg_created, message_id in rows: + msgs.append(ConversationMessage(role=role, content=content or '', created_at=str(msg_created), metadata=_json_loads(msg_meta.read() if hasattr(msg_meta,'read') else msg_meta,{}))) + input_text=next((m.content for m in msgs if m.role in ('user','human')), '') + output_text=next((m.content for m in reversed(msgs) if m.role in ('assistant','ai','agent')), '') + records.append(ConversationRecord(session_id=session_id, trace_id=session_id, message_id=rows[-1][4] if rows else None, agent_id=agent_id, channel=channel, input_text=input_text, output_text=output_text, messages=msgs, metadata=_json_loads(meta.read() if hasattr(meta,'read') else meta,{}), raw={})) + return records diff --git a/evaluator/collectors/base.py b/evaluator/collectors/base.py new file mode 100644 index 0000000..cc00c1f --- /dev/null +++ b/evaluator/collectors/base.py @@ -0,0 +1,8 @@ +from __future__ import annotations +from abc import ABC, abstractmethod +from datetime import datetime +from evaluator.core.models import ConversationRecord + +class ConversationCollector(ABC): + @abstractmethod + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None = None, limit: int | None = None) -> list[ConversationRecord]: ... diff --git a/evaluator/collectors/langfuse.py b/evaluator/collectors/langfuse.py new file mode 100644 index 0000000..95fabfc --- /dev/null +++ b/evaluator/collectors/langfuse.py @@ -0,0 +1,355 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +import httpx + +from evaluator.collectors.base import ConversationCollector +from evaluator.config.settings import settings +from evaluator.core.models import ConversationMessage, ConversationRecord +from evaluator.identity.resolver import IdentityResolver +from evaluator.config.settings import settings + +def _iso_z(dt: datetime) -> str: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _metadata(obj: dict[str, Any] | None) -> dict[str, Any]: + if not isinstance(obj, dict): + return {} + meta = obj.get("metadata") or {} + return meta if isinstance(meta, dict) else {} + + +def _first_value(*values: Any) -> Any: + for value in values: + if value not in (None, "", [], {}): + return value + return None + + +def _content_to_text(value: Any) -> str: + """Convert Langfuse/OpenAI-style content payloads to plain text.""" + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, (int, float, bool)): + return str(value) + if isinstance(value, list): + parts: list[str] = [] + for item in value: + text = _content_to_text(item) + if text: + parts.append(text) + return "\n".join(parts) + if isinstance(value, dict): + # OpenAI multimodal content often comes as {"type":"text","text":"..."} + for key in ("text", "content", "message", "value", "input", "output", "completion"): + if key in value: + text = _content_to_text(value.get(key)) + if text: + return text + # Chat completion response variants. + choices = value.get("choices") + if isinstance(choices, list) and choices: + return _content_to_text(choices[0]) + msg = value.get("message") + if isinstance(msg, dict): + return _content_to_text(msg.get("content")) + return "" + return str(value) + + +def _messages_from_value(value: Any, default_role: str) -> list[ConversationMessage]: + """Extract chat messages from strings/lists/dicts returned by Langfuse.""" + if value in (None, "", [], {}): + return [] + + if isinstance(value, str): + text = value.strip() + return [ConversationMessage(role=default_role, content=text)] if text else [] + + if isinstance(value, dict): + # Common wrappers: {"messages": [...]}, {"input": ...}, {"output": ...} + for key in ("messages", "conversation", "chat"): + if isinstance(value.get(key), list): + return _messages_from_value(value[key], default_role) + + if "role" in value and "content" in value: + role = str(value.get("role") or default_role) + content = _content_to_text(value.get("content")).strip() + return [ConversationMessage(role=role, content=content, metadata={"source": "langfuse"})] if content else [] + + text = _content_to_text(value).strip() + return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else [] + + if isinstance(value, list): + out: list[ConversationMessage] = [] + for item in value: + out.extend(_messages_from_value(item, default_role)) + return out + + text = _content_to_text(value).strip() + return [ConversationMessage(role=default_role, content=text, metadata={"source": "langfuse"})] if text else [] + + +def _agent_id(trace: dict[str, Any], detail: dict[str, Any] | None = None) -> str | None: + detail = detail or {} + meta = {**_metadata(trace), **_metadata(detail)} + return ( + meta.get("agent_id") + or meta.get("agentId") + or meta.get("agent") + or detail.get("name") + or trace.get("name") + ) + + +def _channel(trace: dict[str, Any], detail: dict[str, Any] | None = None) -> str | None: + detail = detail or {} + meta = {**_metadata(trace), **_metadata(detail)} + return meta.get("channel") or meta.get("channel_id") or meta.get("channelId") + + +def _observation_sort_key(obs: dict[str, Any]) -> str: + return str( + obs.get("startTime") + or obs.get("start_time") + or obs.get("createdAt") + or obs.get("created_at") + or obs.get("timestamp") + or "" + ) + + +class LangfuseCollector(ConversationCollector): + """Collect traces from Langfuse and hydrate each trace with detail/observations. + + The list endpoint often returns only trace metadata. If we judge that directly, + prompts reach the LLM as empty conversations and the judge correctly returns + "Conversa vazia"/"Resposta vazia". This collector therefore fetches each trace + detail and observations before building ConversationRecord. + """ + + def __init__(self): + self.identity_resolver = IdentityResolver(settings.identity_config_path) + + async def collect( + self, + period_start: datetime, + period_end: datetime, + agent_aliases: set[str] | None = None, + limit: int | None = None, + ) -> list[ConversationRecord]: + if not settings.can_use_langfuse: + raise RuntimeError( + "Langfuse disabled or credentials missing. Set ENABLE_LANGFUSE=true and LANGFUSE_PUBLIC_KEY/SECRET_KEY." + ) + + params = { + "fromTimestamp": _iso_z(period_start), + "toTimestamp": _iso_z(period_end), + "limit": limit or 100, + } + auth = (settings.langfuse_public_key, settings.langfuse_secret_key) + aliases = {a for a in (agent_aliases or set()) if a} + + async with httpx.AsyncClient(base_url=settings.langfuse_host, timeout=60) as client: + response = await client.get("/api/public/traces", params=params, auth=auth) + if response.status_code >= 400: + raise RuntimeError(f"Langfuse traces API failed {response.status_code}: {response.text}") + payload = response.json() + traces = payload.get("data") or payload.get("traces") or [] + + records: list[ConversationRecord] = [] + for trace in traces: + if not isinstance(trace, dict): + continue + trace_id = trace.get("id") + detail = await self._fetch_trace_detail(client, trace_id, auth) if trace_id else {} + observations = await self._fetch_observations(client, trace_id, auth) if trace_id else [] + + agent_id = _agent_id(trace, detail) + if aliases and agent_id and agent_id not in aliases: + continue + + record = self._to_record(trace, detail, observations, agent_id) + # Do not send empty traces to the LLM judge. Empty records produce + # valid but misleading scores such as "Conversa vazia". + if not (record.input_text or record.output_text or record.messages): + continue + records.append(record) + + return records + + async def _fetch_trace_detail( + self, + client: httpx.AsyncClient, + trace_id: str, + auth: tuple[str | None, str | None], + ) -> dict[str, Any]: + # Langfuse versions differ slightly. This endpoint works in current public API; + # if unavailable, the collector falls back to the list payload. + response = await client.get(f"/api/public/traces/{trace_id}", auth=auth) + if response.status_code >= 400: + return {} + payload = response.json() + if isinstance(payload, dict): + return payload.get("data") if isinstance(payload.get("data"), dict) else payload + return {} + + async def _fetch_observations( + self, + client: httpx.AsyncClient, + trace_id: str, + auth: tuple[str | None, str | None], + ) -> list[dict[str, Any]]: + # Try common Langfuse public API shapes. Ignore failures because trace detail + # may already contain observations in some versions. + candidates = [ + ("/api/public/observations", {"traceId": trace_id, "limit": 100}), + ("/api/public/observations", {"trace_id": trace_id, "limit": 100}), + ] + for path, params in candidates: + response = await client.get(path, params=params, auth=auth) + if response.status_code >= 400: + continue + payload = response.json() + items = payload.get("data") or payload.get("observations") or [] if isinstance(payload, dict) else [] + if isinstance(items, list): + return [x for x in items if isinstance(x, dict)] + return [] + + def _to_record( + self, + trace: dict[str, Any], + detail: dict[str, Any], + observations: list[dict[str, Any]], + agent_id: str | None, + ) -> ConversationRecord: + meta = {**_metadata(trace), **_metadata(detail)} + trace_id = trace.get("id") or detail.get("id") + session_id = ( + detail.get("sessionId") + or detail.get("session_id") + or trace.get("sessionId") + or trace.get("session_id") + or trace_id + ) + + # Detail payload may already include observations. + detail_observations = detail.get("observations") or [] + if isinstance(detail_observations, list): + observations = [*observations, *[x for x in detail_observations if isinstance(x, dict)]] + observations = sorted(observations, key=_observation_sort_key) + + input_value = _first_value( + detail.get("input"), + trace.get("input"), + meta.get("input"), + meta.get("user_message"), + meta.get("message"), + meta.get("question"), + ) + output_value = _first_value( + detail.get("output"), + trace.get("output"), + meta.get("output"), + meta.get("response"), + meta.get("answer"), + ) + + messages: list[ConversationMessage] = [] + messages.extend(_messages_from_value(input_value, "user")) + messages.extend(_messages_from_value(output_value, "assistant")) + + for obs in observations: + obs_meta = _metadata(obs) + obs_kind = str(obs.get("type") or obs.get("name") or "observation").lower() + source_meta = {"source": "langfuse_observation", "observation_id": obs.get("id"), "observation_type": obs_kind} + if obs_meta: + source_meta["metadata"] = obs_meta + + # Prefer preserving explicit chat roles from observation input. + before = len(messages) + messages.extend(_messages_from_value(obs.get("input"), "user")) + for m in messages[before:]: + m.metadata.update(source_meta) + + before = len(messages) + default_output_role = "assistant" if obs_kind in {"generation", "span", "event", "observation"} else "assistant" + messages.extend(_messages_from_value(obs.get("output"), default_output_role)) + for m in messages[before:]: + m.metadata.update(source_meta) + + messages = self._deduplicate_messages(messages) + + input_text = _content_to_text(input_value).strip() + output_text = _content_to_text(output_value).strip() + + if not input_text: + first_user = next((m.content for m in messages if m.role.lower() in {"user", "human"} and m.content), "") + input_text = first_user.strip() + if not output_text: + last_assistant = next( + (m.content for m in reversed(messages) if m.role.lower() in {"assistant", "agent", "ai"} and m.content), + "", + ) + output_text = last_assistant.strip() + + raw = {"trace": trace, "detail": detail, "observations": observations} + + identity_payload = { + **(trace.get("metadata") or {}), + **(trace.get("input") or {}), + "business_context": (trace.get("input") or {}).get("business_context") or {}, + "session_id": trace.get("sessionId") or trace.get("id"), + "message_id": (trace.get("input") or {}).get("message_id"), + "conversation_key": (trace.get("input") or {}).get("conversation_key"), + } + + business_context = self.identity_resolver.resolve(identity_payload) + + metadata = { + **(trace.get("metadata") or {}), + "business_context": business_context, + "ura_call_id": business_context.get("interaction_key"), + } + channel = ( + trace.get("metadata", {}).get("channel") + or trace.get("input", {}).get("channel") + or trace.get("input", {}).get("metadata", {}).get("channel") + or "web" + ) + + return ConversationRecord( + trace_id=trace_id, + session_id=business_context.get("session_key") or trace_id, + message_id=business_context.get("interaction_key") or trace_id, + agent_id=agent_id, + channel=channel, + input_text=input_text, + output_text=output_text, + messages=messages, + metadata=metadata, + raw=raw, + ) + + def _deduplicate_messages(self, messages: list[ConversationMessage]) -> list[ConversationMessage]: + out: list[ConversationMessage] = [] + seen: set[tuple[str, str]] = set() + for msg in messages: + content = (msg.content or "").strip() + if not content: + continue + key = (msg.role.lower(), content) + if key in seen: + continue + seen.add(key) + msg.content = content + out.append(msg) + return out diff --git a/evaluator/collectors/mock.py b/evaluator/collectors/mock.py new file mode 100644 index 0000000..d6efbe5 --- /dev/null +++ b/evaluator/collectors/mock.py @@ -0,0 +1,9 @@ +from __future__ import annotations +from datetime import datetime +from evaluator.collectors.base import ConversationCollector +from evaluator.core.models import ConversationRecord, ConversationMessage + +class MockCollector(ConversationCollector): + async def collect(self, period_start: datetime, period_end: datetime, agent_aliases: set[str] | None=None, limit: int | None=None): + agent = next(iter(agent_aliases), 'telecom_contas') if agent_aliases else 'telecom_contas' + return [ConversationRecord(trace_id='mock-trace-1', session_id='mock-session-1', message_id='mock-message-1', agent_id=agent, channel='web', input_text='quero minha fatura', output_text='Sua fatura está em aberto no valor de R$ 120.', messages=[ConversationMessage(role='user', content='quero minha fatura'), ConversationMessage(role='assistant', content='Sua fatura está em aberto no valor de R$ 120.')], metadata={'mock': True})] diff --git a/evaluator/config/.DS_Store b/evaluator/config/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..bb02fe065de505e77b594c8af7b89d8892a0cd14 GIT binary patch literal 6148 zcmeHKu}%Xq47H)dNtIBCN-$<;s`!CWU5C#7fP-EU9a`Nj3~ZVBBz}RJpJ3!0`W&0W ziK7b=LX|Bg&q-{@_oPc=BI4=8YC<$3q5@5jMd=Zl?p-=D;{lL!jGAt#rZwGB)f4DH zPD$=VWLr^7kI4Nq|7vqTS=7z8)wD04f41BGgd4{&IcDdbPK1=5JSL z2d&%n*DNB!x;O*QfHU9>I0L_80C%=XF;w*48E^)ifjt9qJ_IzuXqXi9(Sa_P06=+0 zXMrxYg!lx*XqXhS0%3Ipsw-QG!Rn6rV0O_kDXKfM6(4Lne-tmAcSrn?&55I;_s)Pb z&}HCMhacMIcCAp9sP>os%E|ro~Q>jv;o_eX;BWR`IZXFe=hblKB4!QQtt`}UImN~qc_ulM# z^X9$x&F+Rmegxx3x$E{f972Cm#u;vV!R)HASV003m{BysU?xV_u2I(ri`fwla~98z zy0IJN+^A>7i@lc2J<5;xurG$D*(mBjf~Oq`-d>k&x7VH!_6t72FYxW?Hg}uBfh?N{ zyr8-SV+JHm%juG?=gpQ1t~Ypcc5vX z5b9^(;B_DkTYKtpz_Qk0?FkAkLL&`mFUXM4gzK`+iHI4voRu|wD5qtV>;|LI%G$N}RLY>4++><0S zCOs((XH+CDH=84~%A9mZ>baV`qd|CS(qo@BgQa=-P7cqB@+Xp7kRZ1I(d_R@S+{L( zCEfW2)9Xatba_7;-Bn>d9E-FADWdKA<}X7JLmT0X>)cjsU^6xV%MJeGE@;tX1YwyT zEb9fVEzB=q$O7SMYXjZ3v}+SmaWE2c^vr!@2vJ`u*^&Z_IYRJw5wyM#ygxhhR!0*)q^xP z>6!>R-|Wu<(}NszYeLt5Mo8nP!j3-42`-w#DVkAhB)QjGVFk{y;+$iu>5y;S|6$8h zgSXY@+_Twoh)rJb{p%d9hkWDTzkxkJZHIgl5P}E&0K9rMchWsUyYRr+s|Sc@?0bZg z>;x_S10(5n{KwGPF9-!7dr9^bO4N;;9t%u}4M?;iC#^0sc?z#r09`Rnn%#7bbVWhS zYBx;dPk8pftZC1^O^GnJV4cgu!(=wv&O{r6L`%jkNb{=cBT_~dDZH6JEK6xom+$H( zo0WCDL1~3yiba@aA9qqhYu3UA2qK@CwKN>&49H9-Z*uf?#9fr*6kVP-*;FP=XdlF^ zVfDTlpbl;5C%6xkv_ZXNI6Jl=2c?-nmt0^#LL_b-)B ztaYt+eRakN#J2+7kFFH2mP6s<`(*%0{vltAuAhHa|K<``scS2@w3Xi4Y&l(Nj+UC& zqN~yM#G^^0?YzV0qx)KsngBJa%uabn;p7_{$d5(76*q!N%g4jZL()5}b8=i+;e=*r&1a zoF?NFAYW{--piV|;Z`C@u{=`%tOqQ|0Ak1t?ER0(1XDP&FdFwzotI`>Rc0_vfZ4Y9 z7IoW&(Cz%KShZp&oMz(yRGa)o`}w$p2SH2WjhqFt!!iu>9CiGG;=iN7a}<4!y0;ns ZE0>=c+(AG9iCuvm%D!q3GmTcre*^N_Bc%WU literal 0 HcmV?d00001 diff --git a/evaluator/config/__pycache__/settings.cpython-313.pyc b/evaluator/config/__pycache__/settings.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d385d1776a9a3efe91816307f6694bcddb0e92b0 GIT binary patch literal 8648 zcmc&(&2Jk?c5jj`i56v1pVo(Ei5B&3S+d8nJ@(AVCPhjjMUgU#8h^=dx5$=6n<6#U zEq^F83uTdH4a`M*SgiMutbtr?BnWiK!ME(6(7gmAtB^s~*j#)F;DzUuQ{F3*O;VyK z3kbnB>{8XQUcGwnRn@EaiXV?3uNC0?xBou#@N%ah{1F?4uPRrueC)Fb!mk7(5X+np zv`~u+>()6dwZdQ9oGmC)kz?XqRj`^?^Lo`>P0&v5!DI9oZ&%ON1|8Jl5|+eXVX5vS z@Wd~AGWWN%yqIr9yGqeqoz$RM)Vbs&_OlMCH7q%RUjsQtYOh#bLc7o_5J#s#>Xz&m zZMiYDajCIaC~&1sOHI*%!<>imkPLRHOtEB*7KloQljDQXTD7iPhwI-XK5StQIi1dE zvX;!G)iAWh1zB5zzg5#oC6ySqvO2G#<NIg#BsvHSDmX^KK_e{3$`Rs3lU(EEY#MYGkX=7s=8o$ zVkI^rUa`@dB|8|cid2)DD^_}p*q3U_F;ps#*K*v!aYqSW$MHIjb0cv2dgA2xanb;M z8cq6o(!}w4jyIcl3u)yzEGW>QFmW4c<9GvU2fV|?>q#fa8#$koC3qLdn>c>T#Jfok z$D28Smx(9Pi@zr4sxy$4_zma}&Qpu5!Gad;#|S z(!{?alN|3M*8sn6;vVAVxQpXcChjACj-TdoPMi1)ndNvd#{(uFBy${hb3A0?^CZmi zK8{CBe1Y8Hct4*vYT|^%I6lDfMH9D?uQ@)**YT!_*OOZuAL4enU4q}?_%O%sB2Jd* z2$$j>r#eGaa=%k}D4H%~lxRdkgq~eGLmqS%oyi!q!DE<|~cE6J3?s+6Z1m05ipyr-lpX*H=OA1h2?c8||1EfRked}~cYS&=tW8f$ngr;-Wo zg(axTDzm8?Wp+8SEN!X^Wpy}e!9&Q+NYuyb5Z{nh^+|>%SPl01NSM_ioKVwY){M|i zPblP%NukJ;Cl-i=|ECsMy1sRVGiil2Bk2=4l~Ob*m5KA2%SM@Vl~R^WIpYt&$*0We4|9T$CpQ8K*$9cTq0l^-A@X&==Z~_wRB9beka-N3A7xb!HxsK0bL2^;10g?Ln>h;HFk1t0exs`6 zC181D?3uitVvQvn2yBViENd(s0{cQ+V@;8%fCOvg2}oWK@k@)*5NpcBlhP_!Jt-~Y zuH2-lQaXRww-5=0W2JOTImOhPH+ z%qgURuaq+sBo6iLC<50$tk0g_nF z#TRfmooKF@e}2Iq^~4sVepa;t!HUM5XfsV&-%zNmZBixqHxPocdZeEB+>&CEIe(Zi z8|cpJk#t>tDruQVaFE|W#Y5lB`3C0wk;Rw=5!kHAG`X&1HZ?f(u%ZcdtD~^6 zdb=;^>rZZVs31X6AkU(GF7ISX(saYb5JG@6X(IW@+jo+&(o zk@X|IUiL~Rfzk_IW@zjNx)r{@oFxRmT+U$pykG<#%jR?3t$Cb zAj)cUn3PD;_n)DiYQ3J=)T~4Tcl@k&S=QoflA7F7-ebVa>d@4l7%oqYusXD{tl^?) zDr?MpjhQJTEiQy29v^GWdyLh%Ds66H%*h&pi@q7ZghbxOsX2d)HE=UaNVB{df23%v zdnOW@0SDz-SOBlWeW3(i!w04hctV7AuVyl<;I8D24RA0yCkzfG0qc~MQ|eZ0!D?`j z0n(o37O$nyEM2O2K3U-V59^?(sY~#Dzu2gEE_6L=5o0x`UNI(4|%+(A2`eabf|q$YzzNU*Ra$2{oHfQ+m_Z} z&;N4%*O6aFo`0j;FaC4;uIF_ZROmw>th~~Yk%7{SLKOZU{0d@QXd84<1&Z8ybFpYM zv{|Dm6<(|uCd?yQ9TeNb+aC9Czx&O1di&^+CFlT#ha_l635x5E5){IK2ugw57^H9y z?ztxn_k`Xxab!6#r#>VH8j&@WSpveF8z@5F5Z%4MP5maNw~ZWGfbL2HU^dUX$^`J_07jhzkKL-TS>r z(@|#d(xCDtlB0xR?V-`Vp|6agul9z5zw6i=ykQL9$PSXd!8^v_9sTb8?4YE(q{C(q zVKg@bp3E}0Bt%Qy7xdy+G>8ma&V?r$IUM+kDi<75KTuGVl#XC=Ku^}bCacBZvE-6r zr%mNQMREris<>Uz^)~PL_F6`bmeIYIz;4Ul`G9dgkaY%id+;x-I=)uXpI1`7Augve zmoe2ZS?_~g_YbK$WV8(JwFF4v%#!$PPfnfWmO$RE&sNXswYvOPo)Zd=0g7j z)$$<&oHJU^fdO`7dl!So#ZMXlw=&=6*P-%foc6-zu5bhj*Oiri4!r19EDr2vdKo}= z($uIJq#QJEK1oVLfBpjE zhV3_;{kpyX$ijFl3FARwdMXH0nu}2&%7+MCq7lP6qT7#tL!YlC(K(c;7^>uRVxD+E zJ|Kt7aJqE6>&TYzSCV52dVfj%R7w3rN!?dcpTW9^ z*(E6r`I{uMT1kQ@kxfWLB~~X%-)zdMe2-It7e$DhQpvQE&PWo)^ot^6X7`7`&L#VG zx%ZS@R!6Il0+S2|gG+wkfx$fR=TUbz61lK&W5I9kV&pQ47{B89`J2Edw{Gz=;NFz` z)$7>HJqJG|e0%bpfyoQS+<{{BPGeYLFh?<%p-WiY$KnANG8D`X*(OAa+B3R_SOyEs zj3_226dfwX=$v9qOnb4wy_CC$pJNTLB#J(rq6?+yx+r=Jibs{=A)&ZGDc=>XlBD8~ zUvYeQpvh&(AXPBZ-y;EThFoTbNe9IO6Cm~9!L|QF*l+HCt?qvQPv8A+R=>AwG{?6? z?^-%u_U>*OE#B?f=kcuQ-ETkja{0}q(H`6m?YOgIaKE|zWz%lbXuiHZ^PFVG>-z&| zc76KX17kqe#kLonSy8S8b)9}StWV!Ex^C-Y%MP%<{SKgh-AjzF*mh(mmK9@_##P|n z(X9_o>Ng%39rE`4&IO#XZ+N#~55|qYgf5J~b>U#`6Qnb983i5}i>>w0+BA^OTQAA|=x+^<9hV)f~PC8O^ivfs*z_x6X!b~Qb`Vhpe9V&{usR$P5|s`u3e z-4iuV5n#2L70G^Q&nufg88$j2+u@z5tQgtv>3c==sar-5n6K>x2!H!sw_6{b(eI>< z?hHz5%ZeGU(7gUYHBM=Hg)|tdpS__co*6x$BpkmH1vO=-Xg8H zNPEp>d(GrJU}6I%cE-fcnAm9(J8fcpCe~+SZWD8x*pP`0nOJUK)N3GjooHvm5WgV(Qt9}+St_>BlSQ+y-lZ&~yoaU7S)Zt_A8$ZFuXrX==R`&&oz&Whpa+P3ctj_0#4wBHMTKUQy9ECbKK{ZYWbe|l;Y t>RP_P3bZ!Io~;M|z3m>@>%MAqU)}cpBKX(A=YREk@#F`aAm$>l{{;n*4we7_ literal 0 HcmV?d00001 diff --git a/evaluator/config/agents.py b/evaluator/config/agents.py new file mode 100644 index 0000000..c080a6a --- /dev/null +++ b/evaluator/config/agents.py @@ -0,0 +1,36 @@ +from __future__ import annotations +from dataclasses import dataclass, field +import yaml +from pathlib import Path +from evaluator.config.settings import settings + +@dataclass +class AgentConfig: + agent_id: str + enabled: bool = True + days_back: int = 1 + percentage: float = 1.0 + langfuse_agent_aliases: list[str] = field(default_factory=list) + gcs_prefix: str = "" + + @property + def aliases(self) -> set[str]: + return {self.agent_id, *self.langfuse_agent_aliases} + + +def load_agents(path: str | None = None) -> list[AgentConfig]: + p = settings.path(path or settings.agents_config_path) + data = yaml.safe_load(p.read_text()) or {} + agents = [] + for item in data.get("agents", []): + cfg = AgentConfig( + agent_id=item["agent_id"], + enabled=bool(item.get("enabled", True)), + days_back=int(item.get("days_back", item.get("daysBack", 1))), + percentage=float(item.get("percentage", 1.0)), + langfuse_agent_aliases=list(item.get("langfuse_agent_aliases", [])), + gcs_prefix=str(item.get("gcs_prefix", "")), + ) + if cfg.enabled: + agents.append(cfg) + return agents diff --git a/evaluator/config/settings.py b/evaluator/config/settings.py new file mode 100644 index 0000000..ce60c4f --- /dev/null +++ b/evaluator/config/settings.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import os +from pathlib import Path +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict +from dotenv import load_dotenv + +ROOT_DIR = Path(__file__).resolve().parents[2] +ENV_PATH = ROOT_DIR / ".env" +load_dotenv(ENV_PATH, override=True) + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=str(ENV_PATH), extra="ignore", case_sensitive=False) + + adb_user: str = Field(default="", validation_alias="ADB_USER") + adb_password: str = Field(default="", validation_alias="ADB_PASSWORD") + adb_dsn: str = Field(default="", validation_alias="ADB_DSN") + adb_wallet_location: str | None = Field(default=None, validation_alias="ADB_WALLET_LOCATION") + adb_wallet_password: str | None = Field(default=None, validation_alias="ADB_WALLET_PASSWORD") + adb_table_prefix: str = Field(default="AGENTFW", validation_alias="ADB_TABLE_PREFIX") + + enable_langfuse: bool = Field(default=False, validation_alias="ENABLE_LANGFUSE") + langfuse_public_key: str | None = Field(default=None, validation_alias="LANGFUSE_PUBLIC_KEY") + langfuse_secret_key: str | None = Field(default=None, validation_alias="LANGFUSE_SECRET_KEY") + langfuse_host: str = Field(default="http://localhost:3005", validation_alias="LANGFUSE_HOST") + publish_langfuse_scores: bool = Field(default=False, validation_alias="PUBLISH_LANGFUSE_SCORES") + + # llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER") + # llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE") + # llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH") + # oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT") + # oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID") + # oci_genai_compartment_id: str | None = Field(default=None, validation_alias="OCI_GENAI_COMPARTMENT_ID") + # oci_genai_auth_type: str = Field(default="api_key", validation_alias="OCI_GENAI_AUTH_TYPE") + # oci_config_path: str | None = Field(default=None, validation_alias="OCI_CONFIG_PATH") + # oci_config_profile: str = Field(default="DEFAULT", validation_alias="OCI_CONFIG_PROFILE") + # llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE") + # llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS") + + # LLM / OCI GenAI OpenAI-compatible, mesmo padrão do Agent Framework + llm_provider: str = Field(default="mock", validation_alias="LLM_PROVIDER") + llm_profile: str = Field(default="judge", validation_alias="LLM_PROFILE") + llm_profiles_path: str = Field(default="configs/llm_profiles/llm_profiles.yaml", validation_alias="LLM_PROFILES_PATH") + + oci_genai_base_url: str | None = Field(default=None, validation_alias="OCI_GENAI_BASE_URL") + oci_genai_endpoint: str | None = Field(default=None, validation_alias="OCI_GENAI_ENDPOINT") # compatibilidade + oci_genai_model: str = Field(default="openai.gpt-4.1", validation_alias="OCI_GENAI_MODEL") + oci_genai_model_id: str | None = Field(default=None, validation_alias="OCI_GENAI_MODEL_ID") # compatibilidade + oci_genai_api_key: str | None = Field(default=None, validation_alias="OCI_GENAI_API_KEY") + oci_genai_project_ocid: str | None = Field(default=None, validation_alias="OCI_GENAI_PROJECT_OCID") + + llm_temperature: float = Field(default=0.0, validation_alias="LLM_TEMPERATURE") + llm_max_tokens: int = Field(default=900, validation_alias="LLM_MAX_TOKENS") + llm_timeout_seconds: int = Field(default=120, validation_alias="LLM_TIMEOUT_SECONDS") + + agents_config_path: str = Field(default="configs/judge/agents.yaml", validation_alias="AGENTS_CONFIG_PATH") + trace_prompt_path: str = Field(default="configs/judge/trace_metrics.yaml", validation_alias="TRACE_PROMPT_PATH") + session_prompt_path: str = Field(default="configs/judge/session_metrics.yaml", validation_alias="SESSION_PROMPT_PATH") + output_dir: str = Field(default="output", validation_alias="OUTPUT_DIR") + batch_size: int = Field(default=50, validation_alias="BATCH_SIZE") + max_attempts: int = Field(default=3, validation_alias="MAX_ATTEMPTS") + enable_gcs_upload: bool = Field(default=False, validation_alias="ENABLE_GCS_UPLOAD") + judge_gcs_bucket: str | None = Field(default=None, validation_alias="JUDGE_GCS_BUCKET") + google_application_credentials: str | None = Field(default=None, validation_alias="GOOGLE_APPLICATION_CREDENTIALS") + identity_config_path: str = "configs/identity.yaml" + + @property + def project_root(self) -> Path: + return ROOT_DIR + + def path(self, value: str | Path) -> Path: + p = Path(value) + return p if p.is_absolute() else ROOT_DIR / p + + @property + def ADB_USER(self): return self.adb_user + @property + def ADB_PASSWORD(self): return self.adb_password + @property + def ADB_DSN(self): return self.adb_dsn + @property + def ADB_WALLET_LOCATION(self): return self.adb_wallet_location + @property + def ADB_WALLET_PASSWORD(self): return self.adb_wallet_password + @property + def ADB_TABLE_PREFIX(self): return (self.adb_table_prefix or "AGENTFW").upper().rstrip("_") + + @property + def has_langfuse_credentials(self) -> bool: + return bool(self.langfuse_public_key and self.langfuse_secret_key) + + @property + def can_use_langfuse(self) -> bool: + return bool(self.enable_langfuse and self.has_langfuse_credentials) + + @property + def can_publish_langfuse_scores(self) -> bool: + return bool(self.publish_langfuse_scores and self.can_use_langfuse) + + @property + def OCI_GENAI_BASE_URL(self) -> str | None: + return self.oci_genai_base_url or self.oci_genai_endpoint + + @property + def OCI_GENAI_MODEL(self) -> str: + return self.oci_genai_model_id or self.oci_genai_model + + @property + def OCI_GENAI_API_KEY(self) -> str | None: + return self.oci_genai_api_key + + @property + def OCI_GENAI_PROJECT_OCID(self) -> str | None: + return self.oci_genai_project_ocid + + @property + def LLM_PROVIDER(self) -> str: + return self.llm_provider + + @property + def LLM_TEMPERATURE(self) -> float: + return self.llm_temperature + + @property + def LLM_MAX_TOKENS(self) -> int: + return self.llm_max_tokens + + @property + def LLM_TIMEOUT_SECONDS(self) -> int: + return self.llm_timeout_seconds + + @property + def LLM_PROFILES_PATH(self) -> str: + return self.llm_profiles_path + +settings = Settings() +if settings.ADB_WALLET_LOCATION: + os.environ["TNS_ADMIN"] = settings.ADB_WALLET_LOCATION diff --git a/evaluator/core/__pycache__/models.cpython-313.pyc b/evaluator/core/__pycache__/models.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3e2b7131b6be7d17213f054350febd14dd66ba7 GIT binary patch literal 3280 zcmbVOOK%fb6rQoKXT}r1lHfd>2Z)dw3Q-@SEwq)=#0hbNlV*Y{6}O|oo`eBsCfpg5 z26bU6Qp?gCB>nQbDm0hK{8&*B%UI!d-sZ>iobI!T%?|kRndlZlB z68vgA7w+haB>joT@egeL$wz;Hd?Xo?AumZenabm+hn7OQ5DjrXycEt6O1MszA~}UB zIhCs1jx1@pD2k~WqFvmtfj@e}ALD)<{9OfQI^60V=iV52 z<89sq_a?xbZ1X0$HwE7AHgAf1d%)Y<=I!1*l}R7sA{m)=S+45^R^WK952jYK0y}WF zY|u&8t!!mNEc&(O+ton5)Ux}06l&M|d$W^H|O zW|=ATGmFdFxhGOS6J?rdy4IF$noKv%Ew5B5qaHKOdljqPYDt>rhC}_J?6|h;nI?_F z8Z?d~fg*`ZrcKkw(-%!E2&i+b64*X;O+nic{3W(Mb;Gx*KUJiTA2^ol-S+(3&RuKA znws-=+_Gnt{3&bGb^~*R!j^VCde^k?Tjh!sc=R1Dulp1|YEQu-?6QApdzT_fxj;<( zJ`g`kjYNNSp^+M{E($H4-kYb`#5;Bl#QV_v-+?ex8r{?YQhhP7w_4FakO28?eh#Cd zudc1kW(@;$0Ut+BHI^1vSF>{*t9(YM-MF0|6ul@;p%_N-9tw^-4x}R}(kKvRaf*J_ z29E{y^>OI&!MjJQIdS@Ux;hUK>@A3KION3XTbU*w<)P)|=_eqM3qZt4XyF>+S_Fux zfL}$cpq!Bks-b-@Qw?&I7&`CM1sFv#G1fclx%Yt$JYBBaz7K&v#DlW1A5cc9SGJj2 z^xOdWkbkLzbNr|Ayyx0XFH##)s{}lIBC}-4-mofV;EYuSn(V$zl$>I46Z(HJ4w>~m z)3)ruf?TvXNS)DT!0@*01=>!)R;Y^mAvhn+pxp`TF>FAAoA+_pFH%E~?=3dsslDaK z$mwdXrq{{o7h`9iu0J2UQoZ)j-6vNB=s2!3=)XeC+n{6i7a;ou+@L933qdtNKL;La zgvu?g)uwSP$}OEnjR^GYYFRqBVrUh*AYd^g4grfB3Fwo27N*vYU`0>I zoSVX)=TUr!;v*CvqnJi<8HEah5nPsjiuMH*oGhQA)`>9}u>l1_;@<-Sj2RvSV%({d zu|}$QZ{;;@+8TyNs|ydmtCP`2-vE#*TqgsK{=w=}t+!4FU!0%%jXXb}tzLVUn5>g* zJKgH!%8Q|~r{ww2banRO=suYi?87~rU|$#<@RN^z0QoliI;1?7fe4Si0?Ds}C0G{) zMxu3`^eVSF&x|>sT_@>A?*N9YZ@5l|8|nV);zPeq`Wt-{B9Ie}KB&vJ z@IDz4p}|T1S7^wqfS@MIg9@S1z&b7g3b#ZFh(WQywm1)B0z#9_q*#w3QnEcbmtafu zOAsvOxEnU5c4^jkL`1@l3lXEX@0Ba~{$XjT+^@>}Dz%FaX87wkQACp^k2Aj&BMvW9 zTV8=P-fVd~kBh%?$z4Mq3f!5G8473%gn14Pk7BO0fjud}o^14u39!cm*f>XLuyN*h z ConversationCollector: + if source == 'langfuse': return LangfuseCollector() + if source == 'agent_framework': return AgentFrameworkCollector() + if source == 'mock': return MockCollector() + raise ValueError('source must be langfuse, agent_framework or mock') + + async def run_agent(self, agent: AgentConfig, period_start: datetime, period_end: datetime, source: str='langfuse', limit: int | None=None) -> dict: + run_id = await self.repository.acreate_run(period_start, period_end, source, agent.agent_id) + try: + await self._emit(run_id, 'RUN_CREATED', f'Agent run created: {agent.agent_id}', agent_id=agent.agent_id, source=source) + collector = self.collector_for(source) + await self._emit(run_id, 'COLLECTING', 'Collecting conversations') + records = await collector.collect(period_start, period_end, agent_aliases=agent.aliases, limit=limit) + await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} records before sampling') + records = self._sample(records, agent.percentage) + await self._emit(run_id, 'SAMPLED', f'Kept {len(records)} records', percentage=agent.percentage) + inserted = await self.repository.ainsert_items(run_id, records) + await self._emit(run_id, 'ITEMS_INSERTED', f'Inserted {inserted} items') + summary = await self._process(run_id) + output_path = export_legacy_txt_gz(self.repository, run_id, agent.agent_id) + await self._emit(run_id, 'EXPORTED', f'Exported {output_path}', output_file=str(output_path)) + return {**summary, 'agent_id': agent.agent_id, 'output_file': str(output_path), 'uploaded_to': None} + except Exception as exc: + await self.repository.amark_run_status(run_id, RunStatus.PARTIAL, str(exc)) + await self._emit(run_id, 'PARTIAL', f'Run failed: {exc}', error=str(exc)) + return {'status':'PARTIAL','run_id':run_id,'agent_id':agent.agent_id,'error':str(exc)} + + async def run(self, period_start: datetime, period_end: datetime, source: str='langfuse', limit: int | None=None) -> dict: + run_id = await self.repository.acreate_run(period_start, period_end, source, None) + try: + collector = self.collector_for(source) + await self._emit(run_id, 'COLLECTING', 'Collecting conversations') + records = await collector.collect(period_start, period_end, limit=limit) + await self._emit(run_id, 'COLLECTED', f'Collected {len(records)} records') + await self.repository.ainsert_items(run_id, records) + return await self._process(run_id) + except Exception as exc: + await self.repository.amark_run_status(run_id, RunStatus.PARTIAL, str(exc)) + await self._emit(run_id, 'PARTIAL', f'Run failed: {exc}', error=str(exc)) + return {'status':'PARTIAL','run_id':run_id,'error':str(exc)} + + async def _process(self, run_id: str) -> dict: + processed_records: list[ConversationRecord] = [] + while True: + items = await self.repository.afetch_next_items(run_id, settings.batch_size) + if not items: break + await self._emit(run_id, 'BATCH_STARTED', f'Processing {len(items)} items') + for item in items: + item_id=item['item_id'] + await self.repository.amark_item_processing(item_id) + try: + raw=item['raw_json'] + if hasattr(raw, 'read'): raw = raw.read() + record = ConversationRecord.model_validate(json.loads(raw)) + result = await self.judge.judge_trace(record) + await self.repository.asave_trace_result(run_id, item_id, record, result) + await self.langfuse_publisher.publish_trace_score(record, result) + await self.repository.amark_item_completed(run_id, item_id) + processed_records.append(record) + + #await self._emit(run_id, 'ITEM_COMPLETED', f'Item completed {item_id}', trace_id=record.trace_id) + loop_result = getattr(result, "loop_result", None) + + await self._emit( + run_id, + "ITEM_COMPLETED", + f"Item completed {item_id}", + trace_id=record.trace_id, + session_id=record.session_id, + judgeScore=result.judgeScore, + accuracyScore=result.accuracyScore, + alucinationScore=result.alucinationScore, + rationale=result.rationale, + loop=getattr(loop_result, "loop", 0) if loop_result else 0, + loop_reason=getattr(loop_result, "reason", "") if loop_result else "", + ) + except Exception as exc: + await self.repository.amark_item_failed(run_id, item_id, str(exc)) + await self._emit(run_id, 'ITEM_FAILED', f'Item failed {item_id}', error=str(exc)) + if processed_records: + sessions = await self.judge.judge_sessions(processed_records) + for sid, result in sessions.items(): + agent_id = next((r.agent_id for r in processed_records if r.session_id == sid), None) + await self.repository.asave_session_result(run_id, sid, agent_id, result) + await self._emit(run_id, 'SESSION_JUDGE_COMPLETED', f'Evaluated {len(sessions)} sessions') + await self.repository.amark_run_status(run_id, RunStatus.COMPLETED) + summary = await self.repository.asummarize_run(run_id) + await self._emit(run_id, 'COMPLETED', 'Run completed', **summary) + return {'status':'COMPLETED', **summary} + + def _sample(self, records: list[ConversationRecord], percentage: float) -> list[ConversationRecord]: + if percentage >= 1: return records + rng = random.Random(42) + return [r for r in records if rng.random() <= percentage] diff --git a/evaluator/identity/__pycache__/resolver.cpython-313.pyc b/evaluator/identity/__pycache__/resolver.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6e1c10699042913a53f5e661c8295ed05e3cf687 GIT binary patch literal 2400 zcmaJ?&2JM&6rcU{M?P$@33e#Ngt*`Y;v^+%qC%yXRsu%}t+$pyX|>weYiGgPb!XPl zaHw*s474h=q$FykgcJ0}pTMz4qNpT{f>a6hz@buTsv`B&H|t%S1llLS&o^ui^2`Nvk`Q<4Sgmrbv@ssXfA>`$A#I+S* zw*)Go9G~@h{>?Tgcu`G${1`$l;G5%nP>P?RTz^WW5~-98n+6SHn8eo1yiUb}k+-Q( z(rvJd12}3 zb4_^eL%R~2e$*P8%WOdalxNBl_X@bJqcAg%K1G;Q5atyQ3yQ4x6rRE50*6I_j^IE? zQba75q*Q>0&$x+a-MpXeh9KL}A;W}&jstM_^JvmV07AJ28gi)Im~EmU%U14?Za`P| zW$d~3EIXVXqHqd7+SubHw;fFb5UI?N>j*@3s?8hM+ictRF&inuKm(r;p@Bz;(Eh6T zYa@u;BKW;kC6tmTu1Hn8cx(jf%&(hyt6-QV>i5t=#f$nZNd+*2CT7V|aaNlt(*V(m zy2^k^16bDzs;z%vQ#o&$MWcjUn0Jh+IH(j)MWUCB&U(VmGLgVxXsBOR4b!kymBx2o zYrkiTfFqd#HHS8$v4!aOnVR@@V56h+w)vg8{K?((ch0XRM^=&}4-P*}jyz0eRy)Sl zIzIjdgImjb?2V+np}w{my7qNR-=P8@tXkt zZz#R%Kxu#x4a!0?J0;tom8NCb?E>|jMyLuuUOBEHQ_6|i1`L|s3YTtNGpy(UvXUno zmwm3oqbOJ}Df~N-4j58O4{)kGM?;O7 z`Wot=(QS?KX3FQFib{s9&yYIS7+X<+*tm{522^l6z-E0FZPvl5U{!&D0ndQOik_zu z<4c{&Zcgl|8To?L!Rd3Ek;Y!ZrfXr~CbvKVKemT!!f)~YOYz0{QrBYFqj>MtGqqEI zwAzR3od=eVEFM`Ny*Kr!bMS_|5j(hicr`Y3Lt2l-mV>`WdLKvQz?l1nfHB|fyWhK- zJaLOG4Bxu)pr>}~`swFwsIzZ#H#(TS+jggIEit%~7`&faO`O>1>Rvv6cl6Hak6%8Q zgnhx9P|I8mZ3a;|QVTj@PKkIwpb;>YdPnV4;ht{ZU&sKo-FoQkrWQxJ5zU?UM zv3*^_I#K__cQJbw*%Lw%pf&}LUfG;{*q-b^kVm3F7hV= dict[str, Any]: + out = {} + + for key, spec in self.keys.items(): + value = None + for source in spec.get("sources", []): + value = _deep_get(payload, source) + if value not in (None, ""): + break + out[key] = str(value) if value not in (None, "") else None + + out["metadata"] = { + "identity_version": self.identity.get("version"), + "identity_source": str(self.path), + } + + return out \ No newline at end of file diff --git a/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc b/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9613a019e91eeb74788fb644d6cd7f04e4f779de GIT binary patch literal 6079 zcmb7ITTB~Q8a`u>?eW-P8w0Tk0Ru_Eq&OEy+N8NG;S#Q<_27txcD=J^fJx1mo*C!D zZq;lmZJer7TJ1(nv=1fHN+D9D(5Ggjl{RfwtG=zhrVE;qw$eW3jWkue-IwlvW;`|! zXw-Q)bIyPM^Pk&)zVAQhNl6Kd;ESByb^5Ubq5qJA@zHvQdh$oeTtyt>sD9K(VJblK zrhXGPX@4^|!{6Lb_gSz-tE2m^eKu^<@|J$4&yMYVEN1&0*rCa+{m#A;T+&yHOSQ7C z-__^FZpb@1rV5>=v4<<+N;x~ppQdn`57i@%twNk*zeyhz_D-0Bu1BO0q*MnlNlArQ z#H1vLNqI~d=Tiw~OpGf~WMn~6L}@|}n$@z7q;y8WvZfpn;z>NFx`#0z7y43T6GB9g zp`BXB39<~^`U)jlvV*pUzgNl`s8REI|MTY-=b@xurO(5ThWk1$5A_=m-! zqN{bZO)l1x9A`RcmZ&JzloyXdf5NlJMMX;1+R<#8jnNu^*;I?Rqj7Y^T#Lr3ARV5u zzWSR`^Q&*dDp|%8{Dh3jyn{BCJ}oCD)k0QIR_$Hq;=(l9M%9D`)hY`-j!&u-r&=a( zGBpi*Kr6yIMWv=>(moJSQ@f)wZ0GJc7G*``rQ~E%o)ll_&+@xFlV>H(?rwep_AEAz z`6=OS62Bf3&hUv8uO#tL)D(426UUHuldaaWq3QFgD@Mi*CX-V!G9v7Q;lK$(@(3gs z(2B`w*}3fUUYU7k=Gyk0t2V>b{&(3?{shq=ca~o1eW&+5akl(+VO`%`nywc}sRjukt!`=9Iv;}M_5@Q3U?$KV#4N2Jiza0{ZJe&Q!HYTQAz5S2p%qG7VL8X6UKlrA2F{h@+vV;H|4` zT4~8NQr8S%LW8daEn$^n*PI$+ua%sdlaOfuuP5{IqMllWlAev`1-+ptwb6+xMXlyD z=&B`v2J6T^1<(Zbycs;xG-Emx2ot};$%LTV;z>yXUjczBvc32X?b&k8y66jLeZkC@=DFZ}_hMUjwyitUdLrZMNz+RX z*L&=Hj*O!^LsvfmizrSK>5vIEFLlah~Sv9IfHMg=0C3 zma}qJYy+%!07~}`a?1IH04S{C@}q5FF1)e(&ucwJsUt;z0}uq@q>MosZX894f^5`c z)c!QUa+JYC(7M>Bm@7DZ#A#S8{yEdC)`S|gZvq9)VYNgdR1}*M6fDMN)g|jZWC(%` zCRnbT0gNz#8%+9CJ$fshd1BMNGO0ROi&U>+XSGV{S|K?EvXB_ZyTRlduV-R0Q4*C{ zO!cqdgOE{GLmC}{1f0oRx#(?KsM?jQYDv@YGRxJCx0%JJfePpkM+} zAm?n|2Q_A>DeP6Tn!wg*fmkCO&?itdwc*}cM#+e^kFy>$gC$!lE13WU33jjhBHncZ zC`L`j$a?|KCSgWrsH^6%#fc>IC^@p)7cEe(L`Ufe0i1OooGZKGX92W{4qJ?L_aQh*)8xEpVkmM8=V!Rv|Mb(xMXK@u6 zS6M9xiAfO5;FeVa<@~guTC`)I*Wn<#6;y|=FOz~6b*UwMJe~raIj<{}8pW8Vqq-qD zii?sKA?q5RRT5q45y4E=jx{A7n9~mNFhoQUWvl2G3YOx%prFwVZXmhsBq8giTEUg&*S;UNy@K)<8#P;t)C~1VzxoDV2q|JwJ8TJOPkMz%ppTZN%0NV#})InQF%T zS(PT%WCDmlcn0O=XRj|y^;7VruZ>nqhABS>^CG0ZR9UrH*_f?t%v3b3AajuV){0DQ zdS{OEWKbDMywnOvm_@2AOO<7i7ewWT$eX3S8B`9U3ay>>Qq{$(boZ5^%R_TJ{tC56 zGU(_EqK;83{*p2}ZC$BG75)!;-tSo~-;pifF*}|sZ&@sDNt>5lHP_^vs{tsMab9s< zc3pGMCKs3k#nQwA6DpR*Zcg5qoO|;wpWFG11*YAoaLtw9@!j%e_8q_9o(py_Femag z8DH~U=bfHgJ(>MIpVj4_8(3fljYf{y;hQ5jM&@3>yDzun=mK+W*;Si$HO_8_B*Qd5 z_8?d22bmz)#rA_;H1)}gRF{eVyiV*14>w(@L1PE!0*bSZH--rOb=mgaRM=J#*nLY^tF(KA< zpB4~XpALRGxckybvH`wt!1RPiqBck=2zD9JDR4v4e8f}C8@VWBti@9tU>_}TM$~R- zNoK(PhHDcW zZq8%$Z~TD+3e3lq0SB;z14g5 zVi@pn*m;R6o`fhw#YWHh%L?rE>dT6F7hDq_I5b=hnlK5vR0^*S)-c?Tkh*UZpvy;m z@&O1cVcRhc8VXl2nS!`&_G_>6l@|P9=R!Ur2{_)3DH7K*}aHKuV zbwqj}Tzc^KP@uCtz;$sPWDl{JOc#ep@**UvE#JH97Nv0kV_~dA7IlItGbFLNEb0uS zQx%H^IhjalPXZNiIXh!K-Mj?%9g#fIsFkb5@cNRM<4)3X7fHHFasrZI2_|u%N+;mz zM(9O*=^hCd{mF9_CAEm6; zK4Vi46HC2$0$YCm_}|-d&-bKlUwC}!_9dtHgMs%4vd+3iXH(YMH0SxZbJt@tD)WC` z>Q0-N>Y8qDyRq$NZPvLh-M8f5_M!i}KU?4V7a>>Qne%tvkKR9-@ee>%Rn1~mOSY;d z;|~G9?bayIdBF*6Rzj`V=t?@>y}}|ez+7ka8{^#0oNw=9+1_-=vbTP=HRo+icP=^I zgbl9^WSU=Ga2_rc=Yk8)wtOj59h#>S-FjL(+Km75?MLeQ-Hh2|l9VcCNeydh|^ z8F%54F!0+zI4KE1T8mXMxo41Tx{|_@9?8&{2ic5|c;o2P?~w zNd!pLidal052*1NufWyw*C|DiL5mz3?M|k7ByJ`}l8~FXTn5SS(RU`ZWt94s-C}9_ zeqV*ftE&?wm^k4i+(!~J6^PKZ2U&3%xFl#TE(_jCYR#ggXnIT@P1Fe=9 zwcv^!(w}xQ2_E!|x35^Qhn{kXJTU^Akt3{QL?g**OSN|SROYb0J4bN}RN$Z@nS`<{ zW{RR7qWUk8;|pYeh#DTE&{ybG7M=PE9eIfMd~3B)JHIQpP}{zDw^8125G2|v{~x@b B7XttQ literal 0 HcmV?d00001 diff --git a/evaluator/judges/llm_judge.py b/evaluator/judges/llm_judge.py new file mode 100644 index 0000000..c69e77e --- /dev/null +++ b/evaluator/judges/llm_judge.py @@ -0,0 +1,81 @@ +from __future__ import annotations +import json +import re +from collections import defaultdict +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, TraceJudgeResult, SessionJudgeResult +from evaluator.llm.client import LLMClient, create_llm_client +from evaluator.prompts.loader import load_prompt + + +def _json_from_text(text: str) -> dict: + try: + return json.loads(text) + except Exception: + m = re.search(r"\{.*\}", text, flags=re.S) + if not m: + raise + return json.loads(m.group(0)) + + +def _history(record: ConversationRecord, max_chars: int = 6000) -> str: + if record.messages: + text = "\n".join(f"{m.role}: {m.content}" for m in record.messages) + else: + text = f"user: {record.input_text}\nagent: {record.output_text}" + return text[-max_chars:] + + +class TIMStyleLLMJudge: + def __init__(self, llm: LLMClient | None = None): + self.llm = llm or create_llm_client() + self.trace_prompt = load_prompt(settings.trace_prompt_path, 'trace_metrics') + self.session_prompt = load_prompt(settings.session_prompt_path, 'session_metrics') + + async def judge_trace(self, record: ConversationRecord) -> TraceJudgeResult: + prompt = f"""{self.trace_prompt} + +HISTÓRICO: +{_history(record)} + +MENSAGEM DO USUÁRIO: +{record.input_text} + +RESPOSTA DO AGENTE: +{record.output_text} + +METADATA: +{json.dumps(record.metadata, ensure_ascii=False, default=str)} +""" + raw = await self.llm.complete(prompt) + data = _json_from_text(raw) + data.setdefault("judge_name", "trace_metrics") + data.setdefault("judge_type", "trace") + data.setdefault("judgeScore", data.get("judge_score", 0)) + data.setdefault("accuracyScore", data.get("accuracy_score", 0)) + data.setdefault("alucinationScore", data.get("alucination_score", 1)) + data.setdefault("rationale", data.get("reasoning", "")) + return TraceJudgeResult(**data) + + async def judge_sessions(self, records: list[ConversationRecord]) -> dict[str, SessionJudgeResult]: + grouped: dict[str, list[ConversationRecord]] = defaultdict(list) + for r in records: + grouped[r.session_id].append(r) + out = {} + for session_id, items in grouped.items(): + transcript = "\n".join(_history(r, 3000) for r in items)[-9000:] + prompt = f"""{self.session_prompt} + +TRANSCRIÇÃO DA SESSÃO: +{transcript} +""" + raw = await self.llm.complete(prompt) + data = _json_from_text(raw) + data.setdefault("judge_name", "session_metrics") + data.setdefault("judge_type", "session") + data.setdefault("inferredCsiScore", data.get("inferred_csi_score", 0)) + data.setdefault("resolution", data.get("resolution", 0)) + data.setdefault("conversationPrecision", data.get("conversation_precision", 0)) + data.setdefault("rationale", data.get("reasoning", "")) + out[session_id] = SessionJudgeResult(**data) + return out diff --git a/evaluator/llm/__pycache__/client.cpython-313.pyc b/evaluator/llm/__pycache__/client.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f3363688bc147371ed74c14aaa7b34ec46a57c7 GIT binary patch literal 5102 zcmb7IO>7&-6`m!RzgH) zlo`uOpTybOH}AdKnfd0uH}kNi#f5Of-Ug0#;q4=OdC8pZo6p5cB5{KJ1#PqVNnck<(IA($w^I35(UPs%5QT>R}(e-$rjDOmPUtL}4pu)TG)H3_ z^zG1{VDWrfmXzEBGHnU5X2P&X2KQTpsE0wQ^dlBbE*TA^kf;}}E7V|DAuLv1QBmJ-}tE5vDCuz_YfVPM5L%59;j9HM?(*W=2CVFh4Y<>F(3Vh6L z7NJt7+8X~KFlm`r#cK_yknukW6$fw(sQd;hRj6=Oih2p&gdTJNHp$wKfrdK4ZnAJz z_@g;nga({CCM_p^T8H5n&IvlMh7N&b*qzXQva%v!EG?hcWKLAEv|8=#{<#Mf)beER zW&lL8@cG|A|C`EYV0e)*jr)PGyD-^Xd9o4p4pCKLmubefyo@DL)(G2)4Tg}GW&!>l zNzw9H;ss5V<<%8~zCXA6mp>Ew$GI@AyRPP!S0&?1^cF!B^H>nqtFJzQvM4KtJ%fV> z;UmHpoxZ9;PqyWJCad8V;F)`-y7C4{f={eV9mWKe+FMW%7+2iAJHe8BaJRj4d+JY9 zl@o)NW1&j;bftG>-)RfA?zvEJ-@e-x=-)%Omi|Yr$QStI#_u*NJ-wA)w!-#Tx`#mU zhzkU}kAa{s1cJ^`ZDYJNO&Opx_6F{NPH`702vDo|Gl0sHS5gq{)PdS+3+*E)s)D5X z=|CNVmL`8@0WSvr4BI#d;J{Y%Yu7ETI@nJIjxE^uVy7hkf8<4^RGq|Gco>^ z87|7Fl9!%f&RmZ1@#q^*@a7jTM2QqQB);YhwaB`VNwaEJ1Xu}amq_4g64|HlqaId= zzf^Bqs&Ar+8ZU*?b>3>#Plc`64hAp@dOBlZ8ju{Oa4Rr0qQ@FYB0fr1JMm$F9?#oGEpUZ_}qLgKtI#uH>1D08| zM0~J8NiwD;GABW?E;-Gnlr>45brW%vDf^5bh!<@lQ%j+LvImPP`-~pQ6gH8mrPvPn za;quv47+6CH<4mL#2)=K>=7T+M2h1Od$t2>MD~6YnVFKp9HMVOpihEl6V>&dhv+*F z=o1IjM0I^visqQ9D-f4L5SNaaViUDWve)D^o>%;wbJ7xn9B~cMdY!s$P1ro>1OiL0 zmtZ3au%1g`txB9bMYp1{Iu0Z%Wv`^Z@cJM+$>7D z`%ucBa->|GZ_=uGQl8|1X}9rAnnoKelq9>jiBdMMEk$wuh-H{cF)1h4K1nx^S7}ML zB#9?#BCeHqm}I1hO?V?e<{dyUI4sDk11AuJ5_jPD3nO}WByaL9NYd)u5gE*{vb8<#f^EqCVAa*ZnI!%H* zxIMBtAgsC?UR-ahdUKwnf7Lj#(UXE$p3SNl@>t^iS@T<_*o_x{?DJV!W3lv3UWQzQ zT~RUHuwK{Pm}uzL!otr{!^ELOcHy4U))jF9;?7JAeEKAC@e~xgqc&2V&O-`KG6t-s zCEWqT&Oso9bmj~ZV<6ksAcNL0$;R-@gonc2c+9Y2#swovxDP}oBZJkgs}f}H2cw|w zZH&eUXu7|_Fhil<*5K8Ktb2)-#pwLyXmVyTm5l1GgfTyJg81UV;b=*q}AxvUOiyI z5=dIs2$68OIY0+;w!?y~c?KaHN&A$lC2xVQ?fI?wav)p`gzpTMPbdC)syvk_P9;i# zg-z$T?ZI2?+wX0?xAUt~=SaCTc5kjc9xIN=O6?annO$G|tr$_?87u`(mjkn(td%3P z#mH>Q_v)st(tdP%dTV;8r_?@FZjaw%$|LdONPLs2_>XKqzxDi%v*bTr_RkTexng8) z(@_bAwliCqo%wrb%46~3SiBUxwCS#N9NT_t>#d!a?{$?&=Zd3qrH6 zdo3t9zK0w^*JI@Hxi;xNKS_q){pj6~XFhtr>>Vh22g=?^(Hkjy$3N{WFys4^D3GG| z2SEA-I_9OpIC5h{3A~TvQC-|%o)dttSznHfb%2^bz`D2No8RbiF-JhO1D)#4Nv@N)gu@eG;a_wNI6K} z?CUD~P8NM9%f8_|!Ls*-qW6W8_sj=#_Z{9sdwxR2NHrC$DxH~mnH#)*t;YlKkJEiVY75Ju_wuZGj{Jx z2uTa#p=!-`Te4lD4ehD~sS-hbVAWlzNR_&Is*u_TyMt=$HruYWm8yM%25F^z>bWx> zncF$(nhBouB?9L3Rt)BufW zKPg)VEy$vOt;ni>8N|TXI%pfPBfGA{3_1p!$XV2PAs6T5Y)#a82DuxkeH3SJqBzG( z7Goa7PFVu3FG(dpV^)ymlq#rFN>+xU&WsA`G<>bS@_d{&dg!?gaw|pA>tSfULB*&Z z`e0F3Lkw$6-EMiVI2Jl#`{QI#?QjfKskN zc9kl4YVw{vi)Zs}YmRNz*tVWQ(m!wMz+&-EWSgu^|}cqd=DF zT89--agv#-XdCqU0JPko-lh=EQGh@i(C6YTI;t4X%~|!5jbk}RFWEUy5pxc%imT=v zr1dM#Rw6?&SY4#BFQuHU7H z19t3GL{*jKDFxT^laib;%|n4&5)3#EJ_tKRS(r$Q30%z+D=q0@SE&d#&nRMY677WP ziWZN^kzyy$OR}W$Jl?tW)I+9_7^2b-85m)M|J|xWT|>Sun5zqFb)gLNuB%YL^Mm>K z=2w_(eGuxNLf45;yFcw-s>?S9bB)23v3qClp4A#(&M-f#eh?)&^1)~u{H7Bc^dB?~ zKhF(nyyPd~7)w7G=oAz8Tpwos;HDP?uU9(o=Z;RiZP5U>_@t7zdZu5rbW*Z4W+|D; z5ik>Llr`61{>CiPs&dUNnX51Lhck#-Mykqt0RLP(qnM=w#x-kxn5U8!#}KWG65BUQ z>>DKe~Bt?V7Td zS*oc0U81d6{8S6YQhsWR3OI(b2mI-IF|P6`l~QpHm^eQg9eK6?bcEwad&f>;YM}<) zatSD*d0{5Ga3G$NC#5N+^J&qBilNtF=xvn*T-Y%sLbRYF9dd|x8ElQuC9qq(losT~ zERYSh3+c2dCon51yf6WTFsq8#E?xo#Qm_LW3CaUD1RjRMjJ^f8G36`*yt{!m5>i4? zMFs5!6@{3$pCldIXcy^rpc1UN`jRHn|KE^ZrPjPV7S#_fzjt|A{v`4H>EBI%d|vbR zW|%c!!YKN za?LDpG-o+hLaJ;TC-dkjsz~{|7$g1QkAigTahB5wsD`?I$m#hTnWf+`9)bxi4~!Hh8!;7Gt$B+Mb=UJyA7 zM9xMbKbVfH60r6{3vR9itPKPlB)xwE*;VRa-kR%k*XHux=A5?~Fci8Sy5F`O%I-U< zdHVob-rDO|-njz7mv?{O8(2merck~8`t-Hw#f9Y~TJ^z}n>8yOCBUtWSfczbz{ zlf7Jok3~=8nsQt3Xg@y~Ig59cn}99{B7I{#w1fjr9i11Y1dx|)CY2DAm<6n*MI@*| zN!UFjT;kQ#1yNS8SCwYO)U3)YB5+nh!OjUm5&2n^#10`X@fXDTGCil_QX*hM#NFtG z8K8B5D#RcSHp3oKzQ`A^fePLWTikR<2tBRvGIUhlhYYaSuy>icQ+2y)<;wkwTEnYr zzFkXWx6a)>mksvZU(kF5YdiNXAGy9w`J2bR@4 zmv3Lr9vytZX?tH^Gsnz!_N|U-4FhZSdzRE&mv3IqhI&_zX!WPocI{sd-x<0+ls!E7 zU`*Q;eO%+(=6*!ETy7nQ!y9fhgg78Lqu2km8Hi=-umUKHfCBAOGJ_!qwkqCcOcyTv zTo0$nOquP`$iw^CDR%3~lsIa_M^K^kS zt0_vS%r|0o^4kpoeV<9hY;B-Cw9TNkGe2)zjs zw1@eTRC#?_}%0A_MTjO&wWX2Kb3DEesEITJ@U{q z`YhE8TKho0ee}VEw)^#mp6KUI`)_sL?EG6(`?Y~gU%^|adHsdv155kguLefj8GPiV zS`RITa?a+0w;}6n&>KOa~2+gceHQ(HJ5b3EPxr$bL*cu`sq#`iW4{CfQMv0Y)h7AkPv>X~MYm z*wy(o#C3?AKRQM-@|vKp3MEO21dqkXh;HLhj-xcR5$q}f$R1f~n*NO1_XWlNopQn7 cXH>&y)DEaVu`_h{SJgdq str: + raise NotImplementedError + + +class MockLLMClient(LLMClient): + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + if "inferredCsiScore" in prompt: + return json.dumps({ + "inferredCsiScore": 0.5, + "resolution": 1, + "conversationPrecision": 1, + "rationale": "Avaliação mock." + }, ensure_ascii=False) + + return json.dumps({ + "judgeScore": 0.7, + "accuracyScore": 0.7, + "alucinationScore": 0.1, + "rationale": "Avaliação mock." + }, ensure_ascii=False) + + +class OCICompatibleLLMClient(LLMClient): + """ + Mesmo padrão do Agent Framework: + - LLM_PROVIDER=oci_openai + - OCI_GENAI_BASE_URL + - OCI_GENAI_API_KEY + - OCI_GENAI_MODEL + - llm_profiles.yaml opcional + """ + + def __init__(self): + self.resolver = LLMProfileResolver(settings) + + async def complete(self, prompt: str, profile_name: str | None = None) -> str: + effective = self.resolver.resolve(profile_name or settings.llm_profile) + + provider = str(effective.get("provider") or settings.LLM_PROVIDER) + model = str(effective.get("model") or settings.OCI_GENAI_MODEL) + base_url = effective.get("base_url") or settings.OCI_GENAI_BASE_URL + api_key = effective.get("api_key") or settings.OCI_GENAI_API_KEY + temperature = effective.get("temperature", settings.LLM_TEMPERATURE) + max_tokens = effective.get("max_tokens", settings.LLM_MAX_TOKENS) + timeout = effective.get("timeout_seconds", settings.LLM_TIMEOUT_SECONDS) + + if provider == "mock": + return await MockLLMClient().complete(prompt, profile_name=profile_name) + + if provider not in ("oci_openai", "openai_compatible"): + raise ValueError(f"Unsupported LLM provider: {provider}") + + if not base_url: + raise RuntimeError("OCI_GENAI_BASE_URL is required for oci_openai provider") + + if not api_key: + raise RuntimeError("OCI_GENAI_API_KEY is required for oci_openai provider") + + from openai import AsyncOpenAI + + client = AsyncOpenAI( + base_url=base_url, + api_key=api_key, + timeout=timeout, + ) + + resp = await client.chat.completions.create( + model=model, + messages=[ + {"role": "user", "content": prompt} + ], + temperature=temperature, + max_tokens=max_tokens, + ) + + return resp.choices[0].message.content or "" + + +def create_llm_client() -> LLMClient: + provider = (settings.LLM_PROVIDER or "mock").lower() + + if provider in ("mock", "none"): + return MockLLMClient() + + if provider in ("oci_openai", "openai_compatible", "oci"): + return OCICompatibleLLMClient() + + raise ValueError(f"Unsupported LLM_PROVIDER={settings.LLM_PROVIDER}") \ No newline at end of file diff --git a/evaluator/llm/profile_resolver.py b/evaluator/llm/profile_resolver.py new file mode 100644 index 0000000..06e5f26 --- /dev/null +++ b/evaluator/llm/profile_resolver.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import copy +import os +import re +from pathlib import Path +from typing import Any + +import yaml + + +def _canonical_profile_name(value: str | None) -> str: + name = (value or "default").strip() + name = name.replace("-", "_").replace(".", "_").replace(" ", "_") + name = re.sub(r"(? Path | None: + root = getattr(self.settings, "project_root", Path.cwd()) + configured = Path(getattr(self.settings, "LLM_PROFILES_PATH", "") or "").expanduser() + candidates = [] + if configured: + candidates.append(configured if configured.is_absolute() else Path(root) / configured) + candidates += [ + Path(root) / "llm_profiles.yaml", + Path(root) / "configs/llm_profiles/llm_profiles.yaml", + Path(root) / "config/llm_profiles.yaml", + Path("llm_profiles.yaml"), + Path("configs/llm_profiles/llm_profiles.yaml"), + ] + for path in candidates: + if path and path.exists() and path.is_file(): + return path + return None + + def _load_profiles(self, path: Path) -> dict[str, dict[str, Any]]: + # Expand ${VAR} placeholders, matching the way env-driven framework config is commonly used. + text = os.path.expandvars(path.read_text(encoding="utf-8")) + data = yaml.safe_load(text) or {} + raw = data.get("profiles", data) + profiles = {} + for name, value in raw.items(): + if isinstance(value, dict): + profiles[_canonical_profile_name(str(name))] = dict(value) + return profiles + + def env_defaults(self) -> dict[str, Any]: + return { + "provider": getattr(self.settings, "LLM_PROVIDER", "mock"), + "model": getattr(self.settings, "OCI_GENAI_MODEL", "mock-llm"), + "temperature": getattr(self.settings, "LLM_TEMPERATURE", 0.0), + "max_tokens": getattr(self.settings, "LLM_MAX_TOKENS", 900), + "timeout_seconds": getattr(self.settings, "LLM_TIMEOUT_SECONDS", 120), + "base_url": getattr(self.settings, "OCI_GENAI_BASE_URL", None), + "api_key": getattr(self.settings, "OCI_GENAI_API_KEY", None), + "project_ocid": getattr(self.settings, "OCI_GENAI_PROJECT_OCID", None), + } + + def resolve(self, profile_name: str | None = None, **overrides) -> dict[str, Any]: + profile_key = _canonical_profile_name(profile_name) + effective = self.env_defaults() + + if self.enabled: + effective.update(copy.deepcopy(self._profiles.get("default") or {})) + effective.update(copy.deepcopy(self._profiles.get(profile_key) or {})) + + for key, value in overrides.items(): + if value is not None: + effective[key] = value + + effective["profile_name"] = profile_key + return effective diff --git a/evaluator/output/.DS_Store b/evaluator/output/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..b72e607cfb57974699266e5b6667f0d340f65eb7 GIT binary patch literal 6148 zcmeHKJ5B>J5SJ@TYmAnzsHGiUQ!^6iFHTAB zGi2LPPq)bZE&pbJvsky?p|`Zp?>@IYbalPicNwfn>i;S=# z!9Xw&3gTfoh?$$45JSQf`MS*l>s>)5}IIkEQWe?pwkinD9>mW=u%6FPjbwT z#SkkHHdmmzvX>Za?wC(*mmQ0txf6Tw!M^iL@xr=0;wLpH&W6zk1Hr(MffF0f<^I3G zFEd)?4_#su3w1Nc^1JoJ=gD20&@RzL#IA?}fj)QyU?AtnsV>@j5*>Eg Wu^37enOApUJOq@G=z@V?VBi@lVl#69 literal 0 HcmV?d00001 diff --git a/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc b/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1709816192a90d0df60440f60189f56094eef8a GIT binary patch literal 13704 zcmc&bTW}lKb$79NFPeSdlAMoZCEUG_bI(2Z+_U#Q?&Z4KY((&U!G2)uqbh`cgA2+-${Auk3Yj+$jcB45 z^$;B4#CcV(ic`Ux>?M2DoLZ_=_iB1HoJPuPdbK?|PM3t$b9#W)_8NMOoKdRN^-?`1 z&Lrjaz2+VZXK|tiL>sA47z^090v>#QW;)CH3hhdB_j)y{%-r;fI zh|44z$EJox0<=HG1w^yY@1Np){%dm42HpKZR>C0Hm?pW<#ALWHz>b7RMK#7F+JfwG zfa3x~9ehxRGI9YvG(Lst74!Td_Qe3lOGqwo0e_Ik)dm=*!#6(OJtP|ZqcFdKl;s0F zZpO6}koAoOx`+G(W<&DPz#CSbSnr1ntT^FGwKB-BLa^%R2?iZW*1D4tqIAb^nxM%e z0EwT2@S_Qw)*zRUe8@V^yaZe_e!N;KM=Pcj@b{d z((l&@_ERzH)CbiusxzwYTnA|6&tym*R7lLM(qm?NFo5*9lk8E-_+6FRuFhs?x<;N|3_?K|rf+$zLs7>zr3cL-L?Xx*4s+a6lC%V-7L0HH|hGpIE*leCc`8VN-V zfhSsb(av59jk)M4-BbFf3{M$(O(Wt}x#%o#WC~Vc^6KO_86Bf%42+Sc>PfHaoE0Go zIT1^-23F7NSmQ3#oBl}Z&7Y&6C1cjsX3giDHEV(m8UCfY30=k)U=JZ z(>cdTSXJ{yQ!eUKodpS>G61Lkm$k8WHiyk+9c&((f01%mr28s4#@d(~k1c3KENsYZ z5#wkh>XINs@NGB|)E3hYwgi5qYoh z&;Bg6r5?QFN)KI3moOH()YG8UFe!TJGS8uOtq0~PKZ-mkE8+uL@F<@O0#3Imkm+r_9aFG+PAIuuR<}|aEn}sU(xbE}Z;6ts$E2bIZO#t$`oXkyFCiVUE23PqP;HFUc?kLRwi#dWsIf4 z#5x(X`;@}h^qY3M+tVc_NAD{}j59Z-CzKKB{aC=1k`s1;b6R;<{5;q_9jS2+D7eyf zX*pm_N~`j2P${|Xlx*M87FIa3u;y?v|ubWz_nNOZ_4jsVi+m-b{i8@D8sl8 z_78wzq?`0{VBGMcK5!K*AaCfhsB7=)>>ud#ij=pn@0_=XIn&kYoz~V~s-37En%311 zJUvj~_005`(V6k^f$@Mp?4&zsy8C>;xBJWiXLtXToo;vM8E*&O?WH@;yE_j!-JSIL zK4t(w4mdl`_dnI?rrUAJd%@k=(T%%q!RmAlgNf=C?VV4x_c1agpkVq2rY~*>gAD=Gw15tJ31JEy_Dp|7HmmQ0#F~*(cLf4G}Y5B zmD~G(fx-HYv+e!;oqafa4!8}AB@G^|ceh{k_R!fmI9=idKWNJ{9vBWg$3j8YDOxkw z+da^EZhFZXbcR?M0heTsa!P(H8kz50{o`<6;fTU}iF0YrOVow9p#bL`yyoQUJKUY^ z10X)_1F0cJ)s^sPnyK*c!6&eL6kz?KA@Gm5LcB=W%!OLYJI04tuVlH4`iVf;2ac6b zG$tJrkO~f)hoUZ7G41f2={(JJIWxVXXQvHeaGe4G-8O9uUz-fnalR|AwrS1fz%{<@ zx7cEzw#YranFDOR@kU-#=g}Pkg104<_*LkFf~D**-Sr+P&Sko z(Hj~kI+r7Fts|&~942lN;Gs$`3Tzds`bG}6CfMO1aXL9{w26jR@QuKy;ZBNsW!DC# zE;%}3;KImsed~D0?;Gb&)~5ikVjTI9{1(liXj#*@2Udu;1|#;Hf#txr2R`L7 z^^@Qr!rlnd7N`%wnv4g$5^fH=Y@(SBaTC7r;B>$XA|mD{*Ubw)u@`1O4k9GxDV19! zh66(uiJD0+$c9BV7BY|+KFIQ69}Fd`hk`)1dK`up)mZFA5~Pa5o`q=Wyy_23N^F$bGTd8D5YiX`~?z+rIe zMQhS?_ku8TK|fS;p)0&d42yb221TO+1N}8p)B%P{1{@&9k&ZQp22KhW_{IYqR^*~F zDgKx-dSIFKy^6XdKA;1*NM&%313L}ayp?s>FO>MtVVhJW16Tc`8r*W8uY~fJjV@b3 z7G6g-iRC5$0p3vfyCn9=@7Rl$%KoP8t*h~hrdUOjU~iu3NvJJxwIil>EDx-liQ?R$ z1eLeEH&PcVUODl@t3SH_kJo?jQoN-<*3$psRiWkS=+R5jOV39OeKE>6bLO7GJYPIl z95Wf4chk~4pK?(&U^S@NEqny2QdHKZzX+;bEz^(^)*^)2>AD-R2fBhlQW zYlv(ytZ6iPrZpq77A$L{=86qUW2js!Ms@q&yZ-j|cVGIwSyy8G#DwaO#%$FK)vd*>dNv&t1Rq(yV&(1TvOy zoJ2;;e9@~#3x}5-E4p7A8duxzj8Dc)qqAotwJU|ICq6KI*b*IhF8cgvBEL4C?~3KS zR`#t93i-!oyY5xh%yuM-OJ{o%Mb4XLk<+t1iPEy!bBSEX?3p!nrFoFJYwOuCp@N#o zSt0-60(q~b{ASCWEs^6wNqxNJ@M_h99wt{*dDHZ!De@(uuxS&KMSBt@)i+z;Y+a#* zlEa%MRNPBe7=)6hO$}5)r_w!Z24pYUu%L>j)#}x&9~2AaeGAkb=aE&v;5@!yzEe;c zsebLN3+nr}yo6(4!ch?^OgNnH)p{n=3bXm9Lj?^^?r>i6p2u6t*n zP6BnJ6xlfl?-8yXdapYcz_Xa1A6_JZ>o@}~|*f1By2b?sG9_}984=TDLUR^C~4 zzCis8bh;Tz|Iwz0mX9>X^A6od1;+EOx{oUD=Ua3iHEE#rqZSRWX(e&~l<|T^wlnj*(BppbZ9>}9oc&*bZ8iLRyuTw4Vf`prH>ZEvDs*USRdUk z1{#PCXpN;;@61+a^e6!I*OYnxrhCd;}8?(Y^qo-Peryd)glHgSt zD@=cAvY$om5V)nmaZw{V zh$1+Sa3Ul-dtR{gxgfxD;E5s0vvsMtPMq(<$vK>0AHZdjP|C;yxHg277jVKsB2r0P z9Xq9>NgmNF4``8_6kk)cCRwr%%=`Oh=jo>z=rEU(C8My1z}Zp4>EG0AqH5R>9h~NnrpJ zvgNER_UzEUGWOXR=CGOy_cwU@DKBs$!<9NXyD;!PF2p zom*}GQOoyR-f#Kfw9woIs;$HRP5Q*Km{y?$stuy0#&oA!&UlZRkYA0Q2RCwxXy@7 z#c`@EMwKm32-Ja13Rjwt2}UT7QRT~D5vYTkW?X66+S`ijY{-;1-!|74E!rng`#0^l zE(g|VGZ*JzwJhreCX;pznKW9i93*W=umco3u^xfV70y0WTPy35nc zCLKwJ&#V!Sxjm`#Q8x}$ZhPPiEfqI^Y&sw^^G9RFTbA_fm>tl|0KJ%I7uk;k6uu!2 zuZYVi>PCHVqae(2*P$%!a-wGNS{T9+uunpSFCfP}AfUtH7!3~Zi5e-sC+&u}2x%jf z5A@|dsS6{*$qM}VHy{B&UT2x*7oLpkN@BW_gvCBTIyZV_Y?i#QvoBP?J{8xM#dKx3 zer#@Rxj?W~DouR4OK>iDSB*;(h5M)K`U_XH-j=zqO4>OejAxMdG#&W9| zRR)v_pi&~8VU0XOcz5W&lr^PL^Et4ljxHXJ6~~D@pnv6u*1*9~c1pEy00vyZ&(4D* zj0MC^*Dr;JIeaK7=~IzPMi8;K-m;Hi_DexiEH@cJ)3e)yrk>=Wa4wb1)j<#ZVAtUs zoXBQn43TxONPVL_7?UzY6?P|sv^~intx5{gl2VWp;*()%di}RF8K%Moph^0EBpc$E zii7zfk(kJg;&O*^U0v27WtIT$&jEVRuILUnt%hFQNSQ-lYNrGcMQ(s4+Jb1d2{H70*8|y z!;fEvWCq67Dqr6CY802(=1pHKY&QYk^2_= z1I)C0cE?h|V!?88#C_Xdx2}Pzb)>dxAAkX*T_g3oDY|N`=n(7=vVdkPyWPdoC#e@% zeBfJ3?FBIVao z`{&_<`%cn11Rg_`F-A&ZK5UE;q(}X&@H1h-=bC_YSB9{QT6{4J^oGWJ#W#ML!`I4O zq)3Ja$3%_PB&s1+KCx9Ez5!4iPGa!mMb@a&p#y~9P#!asf9xojJ%eEeUi&gwk-s!m0Zd-u z8i|Y*4@hLLc>rc{^{0F#jP>$~_LIav6Ya;f|8fX&KWQM^TeUxFq;S5KfON|SrG_5B zf91wp`4+A#o5W5HL4%fnL*~?`1&7Rl1s@@gqz;W;DtVv8x8=Ye;l2n%xl|nfK171e z+%80$HyrXxcMyXw!Vwt&|1Cg$3_l+4F0d)6)EfI8lXXFJqi2>#IP#W>*E(lki0Ylo zIQhRjlmjGKWB*U94T5tT`)541US7^AKyMHQXvtVZzC+ZITV{@>QJVPz%;F8Wxw+*O zE478fB_ZN0@RbGa2a*hzN4CU#RM{k-*`d|o>X~}WhQ0@D+H4o$n3Pt7ZvjqN@VCHZ z9VNpg)Ob=RBct)`ZW6Y^RWmZnEHr<{S)}Hhp{&3R-gVZgi+!v%W7gW;XDunv`@76s z`=^+B7jdp8xss`iVm`1&JSpRxN$rR=a9vTdFr{S2U-(9vgVw}O*r`t zZq^d`Q%~4)XL{~i9SijQ^enlq(m0H>9dPzy2gUhb!P^BZx$l$;w))xjUt4XnFGvRm zl`&gor1Dp`ng@1(2ImeGp>9#PtOGM)R)fumz*_@xswzfReO&PT0(r+#6Jg%F^!B9{ z{+;Ip$KeIy*SU^`LkUND+)*2I)JA;2a@0N0p+X-4UT9J2P2HQih(#!Dh!^%Io%T~g zbMLGfqn>ydP=k-TFNONqctZUKkr_3?sQg3Pw!8ek`7)3IB%d7UV~YQ21$l#Y>r{Y0`eoK3=vzv4LDJObnzD7mtfM>J&MccaIzngX$s#~ab9=!hgi6Yhii-w zuTd(b&QjD`ozYRMg<}Y8XQuPTqS5Oeo&w4OUM~k0@oC!ukAoMemFQ%ulx;-e1*uFZqK~d!=?VDMq)rF8XxN$zZQmDF!j=3 z+Q^r7v}Zfp&vd$_`!>>r5jhOrh3V+VNqQ@kl;#4i!%KY&NkJb;pNjg{iO|s0c;F=W zPtXKHfcR%20h5>@?xL#SAnGnEyo<{2qWq5$^*^ZoE^5Dv8b491jj9>#hJ_%WBsTOz u1Y!GqA6Z9eKN%qGMB~OD2V@WEi0Tc8hOlqgN}=wgfoS|)zJZW~r~eOt5JXP^ literal 0 HcmV?d00001 diff --git a/evaluator/output/legacy_exporter.py b/evaluator/output/legacy_exporter.py new file mode 100644 index 0000000..7fb3824 --- /dev/null +++ b/evaluator/output/legacy_exporter.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import gzip +from pathlib import Path +from datetime import datetime +from typing import Any +import json + +from evaluator.config.settings import settings +from evaluator.persistence.repository import EvaluationRepository +from evaluator.analytics.vloop import vloop_flag + +HEADER = [ + "judgeScore", "accuracyScore", "alucinationScore", "promptLength", "loop", + "inferredCsiScore", "resolution", "conversationPrecision", + "uraCallId", "channelId", "sessionId", "messageId" +] + + +def _q(v) -> str: + return '"' + str("" if v is None else v).replace('"', '""') + '"' + + +def export_legacy_txt_gz(repo: EvaluationRepository, run_id: str, agent_id: str) -> Path: + output_dir = settings.path(settings.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / f"AGENTE_{agent_id}_LLM_JUDGE_{datetime.now().strftime('%Y%m%d')}.TXT.GZ" + + with repo.store.connect() as conn: + cur = conn.cursor() + + cur.execute(f""" + select SESSION_ID, INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION + from {repo.store.t('EVALUATION_RESULT')} + where RUN_ID = :run_id + and JUDGE_TYPE = 'SESSION' + """, {"run_id": run_id}) + + session_metrics = { + sid: { + "inferredCsiScore": csi, + "resolution": res, + "conversationPrecision": prec, + } + for sid, csi, res, prec in cur.fetchall() + } + + cur.execute(f""" + select r.TRACE_ID, r.SESSION_ID, r.JUDGE_SCORE, r.ACCURACY_SCORE, + r.ALUCINATION_SCORE, r.RATIONALE, i.CHANNEL, i.MESSAGE_ID, i.RAW_JSON + from {repo.store.t('EVALUATION_RESULT')} r + left join {repo.store.t('EVALUATION_ITEM')} i on i.ITEM_ID = r.ITEM_ID + where r.RUN_ID = :run_id + and r.JUDGE_TYPE = 'TRACE' + order by r.CREATED_AT + """, {"run_id": run_id}) + + rows = cur.fetchall() + + with gzip.open(path, "wt", encoding="utf-8") as f: + for trace_id, session_id, judge, accuracy, alucination, rationale, channel, message_id, raw_json in rows: + session = session_metrics.get(session_id, {}) + + raw: dict[str, Any] = {} + ura_call_id = "" + channel_id = channel or "" + prompt_length = 0 + loop = 0 + + try: + from evaluator.persistence.oracle_store import _json_loads + + # raw = _json_loads( + # raw_json.read() if hasattr(raw_json, "read") else raw_json, + # {}, + # ) + raw = normalize_raw(raw_json) + + metadata = raw.get("metadata") or {} + + channel_id = ( + metadata.get("channel_id") + or metadata.get("channelId") + or metadata.get("channel") + or channel_id + ) + + ura_call_id = extract_ura_call_id(raw, metadata, message_id) + prompt_length = extract_prompt_length(raw) + loop = vloop_flag(raw) + + # print( + # "[DEBUG promptLength]", + # "trace_id=", trace_id, + # "type(raw)=", type(raw), + # "keys=", list(raw.keys())[:20] if isinstance(raw, dict) else None, + # "prompt_length=", prompt_length, + # "input_text_len=", len(str(raw.get("input_text") or "")) if isinstance(raw, dict) else None, + # "messages=", len(raw.get("messages") or []) if isinstance(raw, dict) else None, + # ) + + except Exception as exc: + print(f"[legacy_exporter] metadata extraction failed trace_id={trace_id}: {exc}") + + vals = [ + judge, + accuracy, + alucination, + prompt_length, + loop, + session.get("inferredCsiScore"), + session.get("resolution"), + session.get("conversationPrecision"), + ura_call_id, + channel_id, + session_id, + message_id or trace_id, + ] + + f.write("|;".join(_q(v) for v in vals) + "\n") + + f.write("|;".join([_q("TOTAL"), _q(len(rows))]) + "\n") + + return path + +def extract_ura_call_id(raw: dict, metadata: dict | None = None, message_id: str | None = None) -> str: + metadata = metadata or {} + + business_context = ( + metadata.get("business_context") + or metadata.get("businessContext") + or raw.get("business_context") + or raw.get("businessContext") + or raw.get("metadata", {}).get("business_context") + or raw.get("metadata", {}).get("businessContext") + or {} + ) + + if not isinstance(business_context, dict): + business_context = {} + + trace = raw.get("raw", {}).get("trace", {}) or raw.get("trace", {}) or {} + detail = raw.get("raw", {}).get("detail", {}) or raw.get("detail", {}) or {} + + trace_input = trace.get("input") or {} + detail_input = detail.get("input") or {} + + trace_metadata = trace.get("metadata") or {} + detail_metadata = detail.get("metadata") or {} + + trace_bc = trace_input.get("business_context") or {} + detail_bc = detail_input.get("business_context") or {} + + return str( + business_context.get("interaction_key") + or business_context.get("ura_call_id") + or metadata.get("ura_call_id") + or metadata.get("uraCallId") + or metadata.get("interaction_key") + or trace_metadata.get("ura_call_id") + or detail_metadata.get("ura_call_id") + or trace_bc.get("interaction_key") + or detail_bc.get("interaction_key") + or message_id + or "" + ) + +def normalize_raw(raw): + if hasattr(raw, "read"): + raw = raw.read() + + if isinstance(raw, bytes): + raw = raw.decode("utf-8") + + if isinstance(raw, str): + raw = raw.strip() + if not raw: + return {} + raw = json.loads(raw) + + # caso esteja duplamente serializado + if isinstance(raw, str): + raw = json.loads(raw) + + return raw if isinstance(raw, dict) else {} + +def extract_prompt_length(raw: dict) -> int: + # 1. tokens reais do Langfuse/framework + tokens = find_prompt_tokens(raw) + if tokens > 0: + return tokens + + # 2. input_size dos spans + input_size = find_input_size(raw) + if input_size > 0: + return input_size + + # 3. fallback garantido pelo ConversationRecord + return ( + len(str(raw.get("input_text") or "")) + + len(str(raw.get("output_text") or "")) + + sum( + len(str(m.get("content") or "")) + for m in raw.get("messages", []) + if isinstance(m, dict) + ) + ) + +def _walk(obj): + if isinstance(obj, dict): + yield obj + for value in obj.values(): + yield from _walk(value) + elif isinstance(obj, list): + for item in obj: + yield from _walk(item) + + +def _to_positive_int(value) -> int: + try: + n = int(value) + return n if n > 0 else 0 + except Exception: + return 0 + + +def find_prompt_tokens(raw: dict) -> int: + candidates = [] + + for obj in _walk(raw): + for key in ( + "prompt_tokens", + "promptTokens", + "input_tokens", + "inputTokens", + ): + n = _to_positive_int(obj.get(key)) + if n: + candidates.append(n) + + usage = obj.get("usage") + if isinstance(usage, dict): + for key in ("input", "prompt_tokens", "promptTokens", "input_tokens", "inputTokens"): + n = _to_positive_int(usage.get(key)) + if n: + candidates.append(n) + + usage_details = obj.get("usageDetails") or obj.get("usage_details") + if isinstance(usage_details, dict): + for key in ("input", "prompt_tokens", "promptTokens", "input_tokens", "inputTokens"): + n = _to_positive_int(usage_details.get(key)) + if n: + candidates.append(n) + + return max(candidates) if candidates else 0 + + +def find_input_size(raw: dict) -> int: + candidates = [] + + for obj in _walk(raw): + for key in ("input_size", "inputSize"): + n = _to_positive_int(obj.get(key)) + if n: + candidates.append(n) + + return max(candidates) if candidates else 0 + +def calculate_text_length(raw: dict) -> int: + return ( + len(str(raw.get("input_text") or "")) + + len(str(raw.get("output_text") or "")) + + sum( + len(str(m.get("content") or "")) + for m in raw.get("messages", []) + if isinstance(m, dict) + ) + ) \ No newline at end of file diff --git a/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc b/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..169842569b09490227162fea43ab25877721224b GIT binary patch literal 15600 zcmd5@Yj6|UmA6Umn zGgVBI+J)Ik0h>@W*-Gu0%~lPwkIMYm+G47_Pysm&pF>Y_tEQ>m1YFbnX9`mzSD)!Z}5xqP;!J={|kuR zMGRtyaWqD71dZdmaUG|Veo2mmUvgYOMsbvsryn+l zg|k4sk})=*i+aw=R52zgR(aM~hguO+(SR89ex2O@e)$J+RZ^Y>@~lc8XPei#Y@c8X z7a>|g@pvK`O2!g#-V5(l;Y2*iUQI5A;-Pt#gOrL$C>aVbhIrnk6HV~KCSyyiXu{!2 zBF+K{>5g9u6Ztmrj{*KjgoyPKh}=c9I8Azd`3@@Y__q}W&;ZeyPl^Bqje20>X7vgc zqw6Evk&E;WLl>-UoKJCVFvN#rG0_-dqoLGd(nX5;i+mz3QjyfsGB1*R5_g`u99m4V zJnkDUzSbS!S&r`xb1^;{3&j%)34S4VDRd>&J(Rc-UrdA|d^e0bo(x90&=Pwk!CeZn zxQ0+N!963VK(h?>L*;Bd%yuU@m^L=ZLs_vu|B`n|q)7Cq8W%6!e4Zkcqe7(G6a((QghdmEH4@&Y&a2NIV;p5QW7V`3eVLr zyNo5%rQ>Xn!ATQ_$41kf9aH9~RHEnbOky&A3WDos&1il7wKdze8&!{uj@vtbP@6S2 zr;W{P_S(0scdT!p&e(U}pmI8r+Wy#9eQV{7mA7|hY_wpaKYwhgSw{r59SXlul_Mec z1&{gl_qA>kJtD1cC;7-hKp2KZ^#QKnhgd%k;azkVaRh@v-3gA|395fq&yXFEBgOP; zjFMsoj$({JVamT5q_+y`&CD64G8XC0%9-X3u1c|LN|G*yO(tXUdH$0IsG6&SK-8x| z$VJmKC@ctYL?k18+`D21eV|{Zy`Zq7?Mi5Ikxd2{6Jbg5)daPS(S5?>^$(vGt;x_F z$nY}9Mq^jsL#_(Z6b!~eH-bUY91JccA}PoYiq>H8>#5LUKF1aeMq?bGT#UupcmlE^ zL4z~FAV=fww&0)*_qjF*Y65);2Sdpu7n@6gOaN6oCUD{abDR5$65zq|-4OgK`n9?8 zTcc}Lj&Dt_Ran0@vevrm`qb^OW%Ro=vAUM)6Svzw(znQU;c7~A?%R-4%sEEK=@}bi zXGjdT7YU9!YhWCV9>1v|Yh;`ZC4sT&Yz1RzKx&Q|!!>7Ns`FrNWoj@`1AtdDwTxMc zRWSh1oQ<(UXlE)p$9$D*n`p`N7lX$qcpg@MEvL}#R5B4yEG1GrJp{-+7vfo3!WcT5 zSX@k80db&{3oPx%Kt>NMKt?aa)UsT>%WMV|W2C2wg7vO#z5;|SkRcxOSu=ZeIl;3LXp~E&<`?J?50tS)S27WV zBsLW3bnT&+Ldo!g(zT^fEKaKs3%$R#$aa~-c@2WWB7}he-ih@l(D%D&#=a;e<6u^$ z3lKiNnyf}P+7gBUIu*}UvQ0KeqAP)Ro9mm!KqNf@f&qiRKi{LGbTIHG)AbCcq;IO3 z@>MD|7k(9K@?mh>@Gu4ihZ$p^A%GUu{sYQmrP=WKur_UYRNHVsvJIt5)odsNuWBh! zr!sAAD_<%CGdPO@TY4CiG76Rm=+v5JTZPqi!=uayjppH>tHLWHvD!U!Jb2nYG2!tC zC#DA7{?RFKQR1|lVNOr^hD1B2^t+Ewc!JYD&+zCB*9!HEM9^j68X?9t;h-4@G!9gl zTIt3m1B}PFhsH6bMmiLaNC`uX7i$E~v0qPtdPQ7SqKVHVw1i%wNvT1!YHJXU_zjAS z#(5TiFv*EjY8jjf(Eu>Y#g;`f9G&0LJJIS(#jzLW;kX0`5FCrgl0iPaz%GTbJH)e# zQSKEOfoKl_W@xi{EU>gTaOrA3*>%wT^zX@^N*`nxj7JXiYm> zvyRTyxwYEHZ0$?w+LyAmJ@@C<>~&c?own0i`>xgFk9V}KjtE_YKlW#Kj1|8$w5)at zdyf3LBGWLIZ8-hYShoIry8e8o{zA5XK3zXAEL_UeFFvU>)|hYT-!$dy$X;{n>dmXS zUc2?$>d?C*_eQeq`{Dls|HGLNW*(k>c=oZgNobzVI8O@JlmBM<%9_=lwKk@$jlZ-u z=SXDTA>n@*Mxs7=9^j8yAA`vA9o2$MS_)7UA#O+S5nh*`!3hg6Hd4!6Vf#egNZ3&21+@&Dg^D7z-^1A0y^2n+j=`t zLUIMmlu+GjWePN~3*fo+QqYd%Xu>RE!@A*t)uXXYUD%`C@RXq}VK&N8mM}q#p^FUY zhKK`1oNPh?@-n(h(D3WSTCm|!#FB<(3Q#=9Jiy{aHAApCfwrZ3Rnt?byihNzE?Zij z+k&#dIY`1H$K>U(zzvIL@TQ`%`5-u11sej|v%$Qy12MJ{iW3IQYk4)xi{>Z`4imWO zbG)o`G|)<>NHpeMwGFk2hHzqODVF5wfKoJZ;0(@%!k0uUya?_Yz!BC!ZW3o2E?oh~ zju-W~CSDIoU<>3W6*r9us*sM2w3?B>{aCO&^!f$a6`NLS&+*^@UX-<(>r zI&u^w{4Qrw>Hdd*2}b_S!QCDc@y}f~LpA6_(gE=gYjzCvkRSGtL;KK&`+C~P1t0McJ3?vh^03TtZ< z^6IhJaf473Y>XK5j<;B$dfs`ii~49P9v1bO=^Qo^8+d*OlEhjq&(#_Nn0glknCp*g zcVugK{i1f)YEq~jxF7jN?ZErdbnQUaHXxV=(zb#B<}ti{v#EnvC3foXQ5{5)Q;#JL zhCrf%QrxQ=o<0~T2umgu+1Us`v3{@7Nxj`S7I$IjZ@;qP|d7+kY8 zydBQiT2^P@AI$7L_~6h_+cJkHAKSNqfd4L60i<8>cqRQ!M-BR(*=->Ive8YDj}U=F zf`WuFW|;C|BPcQNA0Yx>MoHEI3y=?}Ha-QG0%WVqD^Mxm%jmjv%AOUCX0gea9%!vX zi5Gs=QY5Z&(`rf;4TkLhHUf>o4E~{6;zm9I9~rNL!wK-DJ701Y<|i*tm|BYS#WYtISaA5|T@vgB(F1gU;jNgKmKF+fHPR*uCf2;F)S%4fLM5U_Ln9d& zkhk+NU~Uu#*bfuQSUf43B%6V^WbpPo2Qis6SO1A*V7VI4@ODsOT=4iy;#3V;#tmDLq@e)S15WeqI?9LQ2Y zuTXR-5}`$c1~y!T8aD7M4;EZ8bHMZiBqXmbvl|hkDwyB!mRTYe^QkBgx5l4@0Qh99 z&e~ejw${~s?~i9~$I`ZA3R>)0O}=~i-sSrjGaY@xuKou*e{9JfJDEOqGINX(j`{_E zKsa??2n2>{2FChrMuJDHbUdaWX(u zqY>a~Trc@|CD;b6LZw(0&?~k8Gi#r!{xq5d>uo`Bej@!V0E}c^@JOTwtP6OPaf;Dv zHgYJXPHb610D_VQq1@CY7*ffGK)hf>TF@SKYa{|Q5!KO9f`)(d%2qh#g*|YG=ahRQ zAnj5FeI6z-;TN5nq~THT(5UxB5!s~2?;9NyZ5pyszh_dc*Cb5)rcOY0Owe-*HgLr% zO_nds)1K2h`!D)mU z`RQ$j35B+|{&o0%+Yc8q+q*ul zqt|Ncf0r{t`WK&Bfb=`(zu@tz{O-2n6!~}U?neD1yB?yC8qLRb#z!4C_ki)y0Rv7s zOyKwc1?fN0k&yBe%6z=qSadvw*Gdw^8DLV%L7buypfh*_GY@u9QhgK(X=PhNPhi!D z^|1t~F{+1^*Oa`61#6>{heOY(Nvm^InbuUugx2s%^Lafo?K2j1v}}(m%G6|5G!|O3 zl%dh~+PaV0LTlw_vQpDpRhiao8k#SE{B})i<;Jh=wc`&Ozf;p%b(!(kXlS-F{@OD2 zY}3@Uy$p?ZWOe&2Ti|RtIjYyRR&E}(qi^_w=J6|<)^?N`Tcd_%EAzNDJt`;JO`6u4 z%k+rW(7bSum=>_~U`0}ZKFhB_3ng59Kpx%9EBo~_=z%4N6^mraTcIrM;>cyeQz%zj zUsM|GyHvVuOD#K#$|^0F@*&I$D1#3HeZ@9^5*#sQWw`~fpxwLDq%HuIB}y2M&yxA& z18AWse)mKW`%nx}EzN-opWlC@d!ADi{H#H6vphq;U-4o0 z=)~_^)(39jnc(15!0Ug}UVD7LDPM3BIu2Z?VP9~r^u$cK8GmrZ1I#<_ar=Yr7vA~Q^Bb5R+Hwt>#JYGH&t1l9bDb!wHlQ+j?!$>W%Ynj0=+NSo&^0*T=IK}j zj=y8cYtPrDQtJ0?0R)i+KB)SN74JFoqVTRzrl+T;jJ6JW{2*pbaEzJqO3bF=XfS+< z?kviP@ie}cRHlwUoRfvM1ufPLkHf+QbWUBS;6+K_CI%|-7lr*X*d@c@MR&^6>w?48 zC~UdF-65V{=}=NPRZ%PhE0uX9jm0DERp`j*Oo@)@=|r4fsma&5hb|1>#ig(iJp@Bl z2757z6rt#r!1u*P#_w|vdWss_yZ=BL<}sk?a0bz)ek@`i<#^8wC>-9_&1Rdt4 zH1NDQhP(YR@zee<7Y}{z)0?$OP5iPo=tZ!w^t5Vq(5~p9B(WfXqFqr2C1quYssb`Y zidBKbs?dzRoHA~Rw4G5TM_-aMlv29VGw|i9&5J0)SYYUcC+Kxgz8L7s#r$WcHO0WY zDLqnVaLVV&8?@3Ca&^zAa}N#%0Bz0$w?H}(7##J=yIIet9)VjrFG4@+9R|+@{E0zk zbPJt<{jaHs0KWh8%xM^$@}Bbe7`fAM5omA}gXJ@e_@|~O;QkM=d@J3Ufftq~Bu(5Y37Q3`;YKvEID2}y# zrGJy9FX=4gk)haI=^_WM=E{^E)m9q7gLdWwnJS|7D{7JV#TVK z2I+x3O0wIcLGB48STueDcL(=YCsq~v1WK|4K%v>CAY`v1_+-%pCEOWtzD@of* z+iNA;zR@--8I?Y*9ad5+e2UHGZpuTeP%W&I0z!e$5&N3fnp)q9em^Sg+@EPWAT%D#A$xCSj&yX}pCU)4Bex4KGLyM}uPp<^IJ9}$|5LhZ+hzO}xAhuc2bCLEj2^qmxXeK`atOO?553$EV+ z_19_Yf4}lU;MmBnhpw$y*X6XN9T5^-R;ZJ{X+90=5GfiuFHbHe$#?D zEeys4gRyUMFp_Z8T5%7p(8C>?9>SP3(%P44>K7V;Yv#jru12k0GyW1&CDmSEg=?>> zbgZ{hja7F{x5=CpIcl@^wzR!%bx?5iJ=ph=edtLgvnsdsb{K+ToA?8U%lWp6K+H z>A$R%6rF2A2Isntpk7)hA(q3jrv`%R%NdZN=84Wh)#Hqs9FBdui=d7?H5e)9Q>T&Y ze(KawbHt}62i5S~U1xOEhE&r$w}=-_o)?hfnM!hI05kTr_~0fDfeP zUN~?b9B<`6dXwe_&xv#oN4`H{IxK&EcZ7QnGQfvL`~(CbjRf&AYW)qW`W3Q#jCOvE zTK*k%e2liiR1rhOQBD4D{IWMWT+hb+fN!`4+?*T@#z?(9SU$@(?`r23Oh+Vtq z_x&>i41fVe$<9_o>7W1p|M};i@BV(=vsjE2T%rHid4_+QqW&5a(xnk1^u3oM@*Rq$ zSbBsSra9Vyan*>5Q^8$5q8?UrYLcfJ(F`*jL*mScc38*hNL)LjA2x7?VIyZ8HgTq5 zGiM&Qa28Ug8z~yLa@JuRXB#f&iXBuX)kLxSYKk@NRtYoUN=~Vq#@m>1(!41c4$nt} z(b@TM#0zQtbTAr<&YlfHT&0r$>4nWe29yX2ENeLJU7i7k3>Uf zS?FUS!W(=GVKHi(;DS@3;f3i_Azvu6Fc;-Z*ia+_1IhB7YTh&s_2xosbe;?GrofrV zd^j+@aQ0k8O3cj%rz2Cs>V_jYbRG7Ibd(rq)Wy&e`D(RVu0MPZS{fuAZ zr~N9wda^{S>MN6CnY*t%6PIFsE&j0@R~oDl@CTq&0&@<^&$LjDR2S7QHkyVNol!Zc z2Fd|{So(yzftsP6jQ8_8qa){<4a1I%I%dPsc?VxKblg4acTXG{_Xd1^Z}2>HmDi#j zN&-hbT@Lr*A@2l<9Ubx=8TSO(3Ab;e%OR$QyikW7_xlEi;6CA=@UvZxiSY^dXyC}i z&=}j5*R$iOZ+viwWrsXs4aXt(k}fg;yjGYUua_pr zo8@!kjhU(OT482z7x0r=J|=I>49Od1Bbs@2B+Bucg@xH^p22VzhZ~buhv#45byM@< zaA+#ZYo`{t$UMjELNA7<7NQ}Zj&k@_@;cZF-Xq?_JVRF0spssF%{5@)z@QO>dJO6? za6-Uqu-k|pqS8*}aN|pp48T+K4T=CtcnYjwg9ecwK>#4Cxa`{#Z=CqniPujiikp_S zsiKls&%AQx)$l9fL{a^cI%O$-^~5VDl9qQjV@veS50fv0mMr ztnQ3gcP6WM#H)9t9L=fvM)-H9)L2q<-(s^^KUGsjO&cwgt$fMyX)DBGXfgSD1cA?c zrEbT-6Y5(}7zfR|DfrIP7lPMU5jAlTX@*buI}`w)ba5u)OCzRCUK7?;0JuH0pFT$K z)o!Q!Di6JfJ^~=*Gyv}N^mo)=#!R(PddjQWtDd3PXfL2MvJDJ?<)WuyyH0aEV2Cv} zdB40S$eoJt+ONHU_Xu=?Z&G-PreNkJ@eakeD66pznnqj@+=dsKvRA!v;=+lf-T9H- zx!Q8AXRRk@AGzWA$UbuOP~1L}w2s8|BXR4hDvVQ3Nf<@vSHNm}f&qgc|t(9N^w51k7iF z^cG^%p=fY+E_b9YF9~3UH3iyi z0^~$6r~vBHy=wicQtIg&6p+;QlwYe1>I#gU25Q1gF=W*eFayF_)2p6L1FuPBGsxvb z)Sq+ZGZR2=>wIZqJ5wEU6$o$0RU*B0zRFBXDF$_`rFiB(Su4eTNbTfYK1WUlB%zwZ9pTqYb3XuRJ_!Fq!0@n8%dJ05kB!1^ zBJmk|MnqPH(HV{>vPc#$heXw4YaZ4@FpXChzhsr+J>+SEzXKB~*t8w15PnJGpeB6o z!6A~v0un^73;8M$A(D88#~OUv?ez|g7Unt9JeGR`#yL76F-ku7SMpHm=KPWHq%t%+ z3z9E^QA%@6QIvWKc@aU}B_u?|nZs5UToY-izNyn7p$pArcq?R}`ncc=`3M~HWlW4> z*G?0!6DKX;b-grS-k6y#Z_Lb=uqnXv2-D!2U>ERuX?TM8xmL)#4KJ8lG>rS)-P_^X z@p)z5_`=*A9KwXfiM(QXVh}XS8DYm0j&`MLxHcd{0v60MHNOy!ws$xkGu-@H$KoG4 zUN{}%LJnacL3@#tRqDm5xMMQxF}BVuXMPV`7|E#;_&Z!6XQ@CQA+M)MBeh`xQvMv0 zyBvI}6gP%}7X!j9j$;ftL|%6kqzJ%aay^)`o}Hf#%@L_7*%iED=*6kfITY$T4H<%q z*9llV!s~^vH8f3#GD3CnYT#ypw-#i&>d=c*5#(iYt;r?uF=nA*0QXRYZ&XD3@+ye7 zBYz+E1`uJTWlQ=kaG@pTZ**She7$>V@ZVK8zNXz!RTS5!$|~L%xiA7OX=`h$)p_;A z%89EdS5C%u4klVXv6dlVNe|I>;7J=R?-x-m+un{|UHsnSm2V`Pccq%zQ!O1=$5+Nv zZC%N>z45lanTS}>?7Vt#<=}c#SF&kWylGdmsW0Bt2d$DVeessQL`#3F%?bbRRG6#F z?^{aj6&r1oy>6MlF!rsd?{rc%O&dEY3t^sNhIN&x%G%4uOUBFQOXgVX?nLFDSjFBA zO0}>1}xtZyXIJsRs8+n_XFpfGj1DrBeYn&0hsr{i7MJFdjefq3!Y zeXGtgM1Q&$ndU<@Wvg7W{*U_=RAu8mN(IYO(e$Uyt5bi{o~YQFs;v7&O;z^(&kZ${ zeE#VUk+&9GeID5huj1YEL-p$SiUxKK?bdvtayudQ5uGVH)lZd zo3xI_^kZ@Bm;!Z*dLY2_^gR!t%|~Rm)qeG6%r?-jnlGQ>p2V4x%ZHpw&gDzfo0$Sx z3k4cfu|=8!RirvqAl8*J;P8|RRA?!6oQbGB(M4pvglPJg|*E6RiYE7cLU<{aniE_Zq!q7u5 z4c7wsq>P!7V!9csFb$R#DQLZmsXQbkV!;jp{V5|$vCglV#&SG}%F3I&fK~|J=7wQ4 zgQ|iw;ucV?&>IPZ;c0O)K$L1*21^1CiOqP*!b8;Gz?!&1;2O z^Je)>cw?qHq0F7O!qOL{7&sOKLM>a>&q;!8BE1P=hseuw&tM<)G)l1NBD@-&Q9+DtBmD)!p%-5}T&Bsq4FUT-u}ovv#=-|tD3 z_IPpq^rwQKLYqwkFV(3>b7{DF6s{kG?=(u)&s*8f#m z`#auv>ENeMK+ADU<7aoeqqvLE}f zEp9)Sv>uD;kHxLWMBI$bt90S#Kyb4c0L~-gW|d#H8E*cv@v;oZZV@XNkXk6HWhIOU zYJH7ME#Wf|vy;C}dP{=71g8=l!pJo9z5zr!!Cn!CU!FqtcFdG^_;mA&RI(8`X za5&P#@^K=1#5sxlan|HP#eYjLYPqKP^GX}Y7+pk7Z|#=Uc(Tz#wDIMDK#!>Siy!4zPX zJ*c$Ae23!OP^|lh_&N$oy9eVPgE71RCh!yf4}qWXC$0XN-XFL675IrX%vyXzm{m|Z zjzn|LB|!v`bQj<;q#8sah+=}AtDw?PNxGe57BZQJa7w&`lAPPG_9M?AgIxgqN^_dfG5M?_}J0WA+QP&h(ddS z3(bIX?KGMZ1c+D-45qTwbqMw{@*+a@<*DjQGvCbA{MmDJA+X`#+Z8_N^is;)q{g=w*Y zc*j7@K5=v6Bm2aM$K&>iq;(>upNLy0M7XAb=F)|*gV18%epI;j>(byUYwgh7O`5yx z?)x>twOq3X$@BsAU@9U2c`@eitaDIFvJ`3`Nn+#QImtzR*2)~ zxxXtijyni1ZFboTP*NkPG8i(xjIP@X^yH8!C2*7bEr@~TgB>#nTTGzj?_rWi`qK<@ z1Qo^vLrBz~Edv2X7GIeo&aL(~bQJlg5Pa!SQXu>QN(zLZ%#f?P;d1Bl&gF$wO{}V0 zgqa2gX+IOd%*s5NS(gVh4-D*A-`Z~+)IH`fa|vHvKFlnZ0YdR@72sTRU)Mj!4x<HU{jW|@n}|&NTlW1)B-G0l%Fap z8U_CLySd$x&s$ko>5xe!0qUjEZf+O(T1mO7??X*;@oH}8Tc4k>#dF)|Pi|`jtn!ki zdk0vIpp!SGSB0B`Gy%#0Te~n$nC)(i;kM$C$rS)Ff@0`^0ALhEW;Cu5{M7J~33E*# zbph^8k@o#FXdU?$1OR6BrdLg`m_WH#RhBBNM2X)UqZdYFjXM%$J7cB2pxoQ#{Mgxj zt$MBcTJ2hGY|r6@^XXXo5s*d>(@qdacDNMEm|@zmVKv*$8`V@r-Fj7XvZ^~?)t#*B zjaT)qmphW>+v4ThlI7j;^6pf1%N+w#Wc*cCvBAWlk|WEw6PJh3Q9pz)SMIg}_A=lb zBhdhP^n@zRWLy)Vu6mQY8lT8J!C2`l%?WC=`e8EtfaW9()nEjztVEwxqLY;vvP$%s zlDsD{gwaAr4pqguDaqiZ*#c*4@|%?GvZNY{6kVZ_()L*CWuaJ;45FIBW(#Nz;hIcq z$&@Uy6m7P^MZ$+~IOl@g@|-M4LT;r(E&~n|;Naoc`wf2M=DM!}>xtiJ&LyoDYjIgK zwCE#A4vW0J@eDeP1;XIq!&?Nql(V5IH#-%PilQ%_3-JuG4id9ia6yCG!6~p_2u{5u zByHfwFf|({&Q3y(fg_3FT!@f>f^}}O%t{F?bN!Bb~(y-ricqrg?j}0jhxkBN@vq#YrWN_REHz7+*xCaOQU<&nYKuVAL2ak9KKNKM= zZ?X;_DRz%SU*fzU#%`E+>gAX;i2_kjph?C;5Y1Q%K|=DxPkc#>gp{&Pwl%igtE7tN#4_66QHbPGZaa6ZhpmdaKGI<34xtT{U^qvS_n z#|vgS!~zE;6x{0&fI&3bc_!geghWyHr1|2!xp64n;7zhQ3AWQ#UUv%YvZLU2!YrHv z2bRLp`@=uB68@%Zw|>$?*6eM#_a7+*53m)BjK zS)RM@NtEwTmV0h|1@ghwAp!1o#&ui8MeVYAbvj||U1HWtt1s?ZeiCYy?qAZAcCog7 zH(CwCtlx^PPe0T3>c0#w4|CkQmxxlb#1AJ z)?~x(c*E{Q!`^JE`ke+FuvNvCIcDRCVs4cTxP}bs9~+EA6^}U=^~W$>1um*2n70x5 zZHm&t{EGh_cT7QP`cv_ar($;B&0QbaeIM?L+kHu^FQ)g!tv*reB$soe7K~6!Gmjm$ zltDa>@>U6PilmKzO?0B2vjA~XN<~N%?h?d;f3#Ji zL~bmtNP&pd1tB={G3pWTArQ}jpXeZaM4~u=gcu+7qvI=h9*ujC5BXSu{5a|x8a#rg z*+6EbDUSR;za$bwTC`lJ)IwB6{zN2lVJb2!5C!vd3rHkD;v5*g&f>uYSn$x418O%Fz9Swa*T0T+th^LB9SJP)cA((h4eB*4ypgfE$h4975*Ltdz1p4>l!0^X3G zp0GJCW9=3QppRvYuV6sso@>b%57x z`FI^rH#oRLSl_;0QGfA#Y}@`DyAl;ot=G3M&#cbf@FeQp>vb*5$79`|o70K9kQpWu#>S($3is1@uY^JHi5+f%}Kpcb=p) zttZ+Abso`X2X)zUDgXw;OM>xA@`8eTY@AQdl`VVi1)a3A>=Lj~PD@N~cJKnR%#rDO z9*Mt}EOUtW#lKR5j}N7}{x$1yHg(_-bvGQ*vb4hD5iq)$|7j0aB3uz8=R6Ih=={XgWyjgt@wl5q3$QWV3~*#C&6lE z5>DhCGS$~YSjv@VIu3Gr*6PwK`3!8qKfvZ)9LR&}g7PXa8=+kLCW4o52R*Aq|AOz%&L2w&mw-LPE)<6KJT}0_-io)@4 zbD~MUHLZaYGt7b_P)wOZ!jW6RD#1~2@H~RR72E;}u^*xv!F)KxtKldmo|%EOyrSUa zV23FP6HHr3@HCf;gDsJTvw&d1M}i!GqwK?#Ri_E@FA|B0_*cPXOjXzUac9r9zO}w< z{cHWPeMb_V!?6x@H5{coQ(v@XB+)q<>i{P|+DrGQdiTED_fFrt{qOY0o?;Wd6S19s zP#qozsiH*>UI{j;6y%k$;^zCsdQ0m@2~}M7hWUc|^`a&B$8fezL(}q(XB9Vm7<)=sjlwT;F>4i)&G`X=;W=* zn=N-Ny0Rk4ybY)a@o);y&Y=qRk1LErt&cey@>ASH`J_WpFl$Hf%>@LE@~yQGw@pFe zVt>4&KW6vdh<;@E-aH?-dy`ggOz(|by$YtfX)@y(iOhh$b1y$Ffqf(ySXvQ*r~uii zSv8iyU4!>-_)kuR!;d-V+KjmY(gn^-QzA{$h!szNQz8!vnS&@G>1>jqSN%*0tCt^W z03#@f5~Ue{%botX;kfa*DZ+@3eQt_c0AO|Qm2-cBipvc0oJ0{Wx%|e=(U$FEUeE?B zW>hi(mc|;V{YE$=NLS{~4Pkv4#EqBB0Pmr8jJp5O({f!6&c-LUuI=G}!P{1i*Wi>eQX|tU8Jc-(&Sj{1@s5vZmmyZ|sqmTT- zje4rN?P}Lb*W0d7torhzyG2yZM?_0 zD?ByoTeWVzr_}s@3j>J{O7$4l7(LF%9OM2Ho%D-dABLR;oTo{TvwS1&eTm~{#?d4>p86Q=U{Cv6X0O* zqcTwrbf|~5%EiBFhkDIscr)!rU+i7HlZ6%#GCl;3lujthsMof9#%tcNGN3F!{`Enp zy(*vPTA+8qzn%zvPbpXV(h)5egY6>_CdhXZEXw@_#%@Az8(;8kEe1La(46x&(w(=_ zHrQzrm_nirYk&;Sfk7h%O&Gif0dEz(BjJ2lau5xneF)wbY(@+7qXP0JI&7e{4-$G$7GisQF^P>@CM)-O~mLgq#5};b72!gFLq4_@R`xCW&v6=&L{$M}& ztP6+3l-6Ag#fn=H-?eRYQPp*qYcJKlRe!I}T2^$oma1-l&Am}YH8_^(OVcSw<8tGQ zX1!(GD!tmcrnz#2;I~zJW%`OURo_7B2)A5La$Cphj@4spyRPlOzT?U_(>+`>T&J&i zWxWVd)k^3}$NdU3AkAu+!X2Cp!I#9#y-Z3G1c)#6O?R26RQ+DhK&hul`+-UYi4Tex z3`>ok#>X80{Wo|31^j0WreA%5BYHCxUc|lOf_@Z^K)mK!bH(h#H^7Kx_$C;!3@5F_ zG5v7dI;_BFB7I5lh?$4?k))6Zf)kEU$*jI_zbTaKLbwKf!Q}fF=D5 zOK>CH2_}TQfB_jnKn!Vu_ux)&8r%uKf;$n-!ku6qxZAK_8BSu@;w+q0;<5lv8cO3N zIEJPmPJ%;TwjjDK^#)OGln0(gv?Rz3i3O?Leq$>Jqj_D5C&UGV zJRqv-GH0UgOH}TURrCRJJCH$cqUaNlTkFPl5xKotzp7oe$E$koH5LPob5PaW0ms!* z%`MCB6|klRnJ3)p>MuTjX(9!gARSG$0mfUIPB{UKt(B!ZI#=Cm>6)&d>-5@msu3iq zD`kixn^qX?WniUMj?HR{0GrjrV0Um>88(wrGHg~cXi>kn(`_8oYj5?dAo0GQ!O&tH zta;3_*(bO$^5v%%5m|{lLRsjZRp%0H3GO$sH&nVhX`&8r(uY9gQVJQo=DGR#IpJ_& zqi_no1}i%-AhAY_5vmLczt4?u`o7~E4lk+=z+gGLz=cUE^mh=-y`XR)hEt@)lj*ik zrDG_^Sv29fMPfIiwnG@>IF?r8o}L_(8E5&((T88<{x^0*s6#)l$KIY!^twlS4=kJ4D3+KdI`Tkk7Rst1z6Yy)qP0M@Ug*D!DZ1N5Gvkn3%NXzW zW8+V>)sVZ7W=2SE(XSxaOAC*IPv9)Ymo>8h-y^&%4v0_D5dptpUJLFVa9jgNumpOG6fHg2Gi{%^8GD>G#0v`-QD`kTXF6{1Zs=4K+>Ezo44_nlk-@s`~|1 h{x_8Cm(=q=r=I^MH4&#KKGk})puoFJksImn{{dxuU<3dF literal 0 HcmV?d00001 diff --git a/evaluator/persistence/oracle_store.py b/evaluator/persistence/oracle_store.py new file mode 100644 index 0000000..fa2d97c --- /dev/null +++ b/evaluator/persistence/oracle_store.py @@ -0,0 +1,282 @@ +from __future__ import annotations + +import asyncio +import json +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any + + +def _json_dumps(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _json_loads(value: str | bytes | None, default: Any): + if value is None: + return default + if isinstance(value, bytes): + value = value.decode("utf-8") + try: + return json.loads(value) + except Exception: + return default + + +@dataclass +class OracleSettings: + user: str + password: str + dsn: str + wallet_location: str | None = None + wallet_password: str | None = None + table_prefix: str = "AGENTFW" + + +class OracleStore: + """Oracle Autonomous Database store following the Agent Framework pattern. + + Uses direct oracledb.connect() with wallet arguments. Synchronous DB calls are + exposed through asyncio.to_thread(), matching the main framework style. + """ + + def __init__(self, settings, auto_init_schema: bool = False): + self.settings = settings + self.cfg = OracleSettings( + user=settings.ADB_USER or "", + password=settings.ADB_PASSWORD or "", + dsn=settings.ADB_DSN or "", + wallet_location=getattr(settings, "ADB_WALLET_LOCATION", None), + wallet_password=getattr(settings, "ADB_WALLET_PASSWORD", None), + table_prefix=(getattr(settings, "ADB_TABLE_PREFIX", "AGENTFW") or "AGENTFW").upper().rstrip("_"), + ) + if not self.cfg.user or not self.cfg.password or not self.cfg.dsn: + raise RuntimeError("ADB_USER, ADB_PASSWORD and ADB_DSN are required") + if auto_init_schema: + self._init_schema() + + @staticmethod + def now() -> datetime: + return datetime.now(timezone.utc) + + def t(self, name: str) -> str: + return f"{self.cfg.table_prefix}_{name}".upper() + + @contextmanager + def connect(self): + import oracledb + oracledb.defaults.fetch_lobs = False + kwargs = {} + if self.cfg.wallet_location: + kwargs["config_dir"] = self.cfg.wallet_location + kwargs["wallet_location"] = self.cfg.wallet_location + if self.cfg.wallet_password: + kwargs["wallet_password"] = self.cfg.wallet_password + conn = oracledb.connect(user=self.cfg.user, password=self.cfg.password, dsn=self.cfg.dsn, **kwargs) + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + async def to_thread(self, func, *args, **kwargs): + return await asyncio.to_thread(func, *args, **kwargs) + + def _exec_ddl_ignore_exists(self, cur, ddl: str): + try: + cur.execute(ddl) + except Exception as exc: + msg = str(exc) + if "ORA-00955" in msg or "ORA-01408" in msg or "ORA-02275" in msg: + return + raise + + def _column_exists(self, cur, table_name: str, column_name: str) -> bool: + cur.execute(""" + select count(*) + from user_tab_columns + where table_name = :table_name + and column_name = :column_name + """, {"table_name": self.t(table_name), "column_name": column_name.upper()}) + return int(cur.fetchone()[0] or 0) > 0 + + def _ensure_column(self, cur, table_name: str, column_name: str, ddl_type: str): + if not self._column_exists(cur, table_name, column_name): + cur.execute(f"alter table {self.t(table_name)} add {column_name.upper()} {ddl_type}") + + def drop_schema(self): + tables = [ + "EVALUATION_RESULT", + "EVALUATION_FINDING", + "EVALUATION_METRIC", + "EVALUATION_ITEM", + "EVALUATION_PROGRESS_EVENT", + "EVALUATION_RUN", + ] + with self.connect() as conn: + cur = conn.cursor() + for table in tables: + try: + cur.execute(f"drop table {self.t(table)} cascade constraints purge") + except Exception as exc: + if "ORA-00942" in str(exc): + continue + raise + + def _init_schema(self): + with self.connect() as conn: + cur = conn.cursor() + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_RUN')} ( + RUN_ID varchar2(64) primary key, + AGENT_ID varchar2(128), + SOURCE varchar2(64), + PERIOD_START timestamp with time zone, + PERIOD_END timestamp with time zone, + STATUS varchar2(32) not null, + TOTAL_ITEMS number default 0 not null, + PROCESSED_ITEMS number default 0 not null, + FAILED_ITEMS number default 0 not null, + RETRY_COUNT number default 0 not null, + ERROR_MESSAGE clob, + LAST_HEARTBEAT_AT timestamp with time zone, + CREATED_AT timestamp with time zone not null, + UPDATED_AT timestamp with time zone not null + ) + """) + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_PROGRESS_EVENT')} ( + ID number generated always as identity primary key, + RUN_ID varchar2(64) not null, + STAGE varchar2(128) not null, + MESSAGE varchar2(1000), + DETAILS_JSON clob check (DETAILS_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_PROGRESS_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_PROGRESS_RUN')} on {self.t('EVALUATION_PROGRESS_EVENT')}(RUN_ID, CREATED_AT)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_ITEM')} ( + ITEM_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + TRACE_ID varchar2(256), + SESSION_ID varchar2(256), + MESSAGE_ID varchar2(256), + AGENT_ID varchar2(128), + CHANNEL varchar2(64), + STATUS varchar2(32) not null, + ATTEMPT_COUNT number default 0 not null, + ERROR_MESSAGE clob, + RAW_JSON clob check (RAW_JSON is json), + CREATED_AT timestamp with time zone not null, + UPDATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_ITEM_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_ITEM_RUN')} on {self.t('EVALUATION_ITEM')}(RUN_ID, STATUS, CREATED_AT)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_RESULT')} ( + RESULT_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + TRACE_ID varchar2(256), + SESSION_ID varchar2(256), + AGENT_ID varchar2(128), + JUDGE_NAME varchar2(128) not null, + JUDGE_TYPE varchar2(32), + SCORE number, + JUDGE_SCORE number, + ACCURACY_SCORE number, + ALUCINATION_SCORE number, + HALLUCINATION_SCORE number, + INFERRED_CSI_SCORE number, + RESOLUTION number, + CONVERSATION_PRECISION number, + TOOL_USAGE_SCORE number, + ROUTING_SCORE number, + RATIONALE clob, + REASONING clob, + RESULT_JSON clob check (RESULT_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_RESULT_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID), + constraint {self.t('FK_EVAL_RESULT_ITEM')} + foreign key (ITEM_ID) references {self.t('EVALUATION_ITEM')}(ITEM_ID) + ) + """) + self._exec_ddl_ignore_exists(cur, f"create index {self.t('IX_EVAL_RESULT_RUN')} on {self.t('EVALUATION_RESULT')}(RUN_ID, ITEM_ID)") + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_METRIC')} ( + METRIC_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + METRIC_NAME varchar2(128) not null, + METRIC_VALUE number, + DIMENSIONS_JSON clob check (DIMENSIONS_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_METRIC_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + + self._exec_ddl_ignore_exists(cur, f""" + create table {self.t('EVALUATION_FINDING')} ( + FINDING_ID varchar2(64) primary key, + RUN_ID varchar2(64) not null, + ITEM_ID varchar2(64), + SEVERITY varchar2(32), + CATEGORY varchar2(128), + TITLE varchar2(512), + DESCRIPTION clob, + EVIDENCE_JSON clob check (EVIDENCE_JSON is json), + CREATED_AT timestamp with time zone not null, + constraint {self.t('FK_EVAL_FINDING_RUN')} + foreign key (RUN_ID) references {self.t('EVALUATION_RUN')}(RUN_ID) + ) + """) + + # Non-destructive compatibility for older generated schemas. + for col, typ in [ + ("RETRY_COUNT", "number default 0"), + ("ERROR_MESSAGE", "clob"), + ("LAST_HEARTBEAT_AT", "timestamp with time zone"), + ("UPDATED_AT", "timestamp with time zone"), + ]: + self._ensure_column(cur, "EVALUATION_RUN", col, typ) + for col, typ in [ + ("ID", "number generated always as identity"), + ]: + # Identity column cannot always be added cleanly; ignore if table is old without ID. + try: + self._ensure_column(cur, "EVALUATION_PROGRESS_EVENT", col, typ) + except Exception: + pass + for col, typ in [ + ("JUDGE_NAME", "varchar2(128) default 'unknown_judge' not null"), + ("JUDGE_TYPE", "varchar2(32)"), + ("SCORE", "number"), + ("JUDGE_SCORE", "number"), + ("ACCURACY_SCORE", "number"), + ("ALUCINATION_SCORE", "number"), + ("HALLUCINATION_SCORE", "number"), + ("INFERRED_CSI_SCORE", "number"), + ("RESOLUTION", "number"), + ("CONVERSATION_PRECISION", "number"), + ("TOOL_USAGE_SCORE", "number"), + ("ROUTING_SCORE", "number"), + ("RATIONALE", "clob"), + ("REASONING", "clob"), + ("RESULT_JSON", "clob"), + ]: + self._ensure_column(cur, "EVALUATION_RESULT", col, typ) diff --git a/evaluator/persistence/repository.py b/evaluator/persistence/repository.py new file mode 100644 index 0000000..e89969b --- /dev/null +++ b/evaluator/persistence/repository.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from evaluator.config.settings import settings +from evaluator.core.models import ConversationRecord, ItemStatus, RunStatus, TraceJudgeResult, SessionJudgeResult +from evaluator.persistence.oracle_store import OracleStore, _json_dumps, _json_loads + + +class EvaluationRepository: + def __init__(self, auto_init_schema: bool = False): + self.store = OracleStore(settings, auto_init_schema=auto_init_schema) + + def create_run(self, period_start: datetime, period_end: datetime, source: str, agent_id: str | None = None) -> str: + run_id = str(uuid.uuid4()) + now = self.store.now() + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RUN')} + (RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS, TOTAL_ITEMS, + PROCESSED_ITEMS, FAILED_ITEMS, RETRY_COUNT, LAST_HEARTBEAT_AT, CREATED_AT, UPDATED_AT) + values (:run_id, :agent_id, :period_start, :period_end, :source, :status, + 0, 0, 0, 0, :heartbeat_at, :created_at, :updated_at) + """, { + "run_id": run_id, + "agent_id": agent_id, + "period_start": period_start, + "period_end": period_end, + "source": source, + "status": RunStatus.RUNNING.value, + "heartbeat_at": now, + "created_at": now, + "updated_at": now, + }) + return run_id + + async def acreate_run(self, *args, **kwargs): + return await self.store.to_thread(self.create_run, *args, **kwargs) + + def record_progress(self, run_id: str, stage: str, message: str = "", details: dict | None = None): + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_PROGRESS_EVENT')} + (RUN_ID, STAGE, MESSAGE, DETAILS_JSON, CREATED_AT) + values (:run_id, :stage, :message, :details_json, :created_at) + """, { + "run_id": run_id, + "stage": stage, + "message": (message or "")[:1000], + "details_json": _json_dumps(details or {}), + "created_at": self.store.now(), + }) + + async def arecord_progress(self, *args, **kwargs): + return await self.store.to_thread(self.record_progress, *args, **kwargs) + + def insert_items(self, run_id: str, records: list[ConversationRecord]) -> int: + inserted = 0 + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + for record in records: + try: + cur.execute(f""" + insert into {self.store.t('EVALUATION_ITEM')} + (ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL, + STATUS, ATTEMPT_COUNT, RAW_JSON, CREATED_AT, UPDATED_AT) + values (:item_id, :run_id, :trace_id, :session_id, :message_id, :agent_id, + :channel, :status, 0, :raw_json, :created_at, :updated_at) + """, { + "item_id": str(uuid.uuid4()), + "run_id": run_id, + "trace_id": record.trace_id, + "session_id": record.session_id, + "message_id": record.message_id, + "agent_id": record.agent_id, + "channel": record.channel, + "status": ItemStatus.PENDING.value, + "raw_json": record.model_dump_json(), + "created_at": now, + "updated_at": now, + }) + inserted += 1 + except Exception as exc: + if "ORA-00001" not in str(exc): + raise + cur.execute(f""" + update {self.store.t('EVALUATION_RUN')} + set TOTAL_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id + ), + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, {"run_id": run_id, "updated_at": self.store.now()}) + return inserted + + async def ainsert_items(self, *args, **kwargs): + return await self.store.to_thread(self.insert_items, *args, **kwargs) + + def fetch_next_items(self, run_id: str, batch_size: int) -> list[dict]: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select ITEM_ID, RUN_ID, TRACE_ID, SESSION_ID, MESSAGE_ID, AGENT_ID, CHANNEL, + STATUS, ATTEMPT_COUNT, RAW_JSON + from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id + and STATUS in (:pending, :failed) + and ATTEMPT_COUNT < :max_attempts + order by CREATED_AT + ) where rownum <= :batch_size + """, { + "run_id": run_id, + "pending": ItemStatus.PENDING.value, + "failed": ItemStatus.FAILED.value, + "max_attempts": settings.max_attempts, + "batch_size": batch_size, + }) + cols = [d[0].lower() for d in cur.description] + return [dict(zip(cols, row)) for row in cur.fetchall()] + + async def afetch_next_items(self, *args, **kwargs): + return await self.store.to_thread(self.fetch_next_items, *args, **kwargs) + + def mark_item_processing(self, item_id: str): + with self.store.connect() as conn: + conn.cursor().execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + ATTEMPT_COUNT = ATTEMPT_COUNT + 1, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.PROCESSING.value, + "updated_at": self.store.now(), + "item_id": item_id, + }) + + async def amark_item_processing(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_processing, *args, **kwargs) + + def mark_item_completed(self, run_id: str, item_id: str): + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.COMPLETED.value, + "updated_at": now, + "item_id": item_id, + }) + self._refresh_run_counters(cur, run_id, now) + + async def amark_item_completed(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_completed, *args, **kwargs) + + def mark_item_failed(self, run_id: str, item_id: str, error: str): + now = self.store.now() + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + update {self.store.t('EVALUATION_ITEM')} + set STATUS = :status, + ERROR_MESSAGE = :error, + UPDATED_AT = :updated_at + where ITEM_ID = :item_id + """, { + "status": ItemStatus.FAILED.value, + "error": (error or "")[:4000], + "updated_at": now, + "item_id": item_id, + }) + self._refresh_run_counters(cur, run_id, now) + + async def amark_item_failed(self, *args, **kwargs): + return await self.store.to_thread(self.mark_item_failed, *args, **kwargs) + + def _refresh_run_counters(self, cur, run_id: str, updated_at): + cur.execute(f""" + update {self.store.t('EVALUATION_RUN')} + set PROCESSED_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id and STATUS = :completed + ), + FAILED_ITEMS = ( + select count(*) from {self.store.t('EVALUATION_ITEM')} + where RUN_ID = :run_id and STATUS = :failed + ), + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, { + "run_id": run_id, + "completed": ItemStatus.COMPLETED.value, + "failed": ItemStatus.FAILED.value, + "updated_at": updated_at, + }) + + def save_trace_result(self, run_id: str, item_id: str, record: ConversationRecord, result: TraceJudgeResult): + judge_name = getattr(result, "judge_name", None) or "trace_metrics" + judge_type = (getattr(result, "judge_type", None) or "TRACE").upper() + score = getattr(result, "judgeScore", None) + accuracy = getattr(result, "accuracyScore", None) + alucination = getattr(result, "alucinationScore", None) + rationale = getattr(result, "rationale", None) or "" + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RESULT')} + (RESULT_ID, RUN_ID, ITEM_ID, TRACE_ID, SESSION_ID, AGENT_ID, JUDGE_NAME, + JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE, ALUCINATION_SCORE, + RATIONALE, RESULT_JSON, CREATED_AT) + values (:result_id, :run_id, :item_id, :trace_id, :session_id, :agent_id, + :judge_name, :judge_type, :score, :judge_score, :accuracy_score, + :alucination_score, :rationale, :result_json, :created_at) + """, { + "result_id": str(uuid.uuid4()), + "run_id": run_id, + "item_id": item_id, + "trace_id": record.trace_id, + "session_id": record.session_id, + "agent_id": record.agent_id, + "judge_name": judge_name, + "judge_type": judge_type, + "score": score, + "judge_score": score, + "accuracy_score": accuracy, + "alucination_score": alucination, + "rationale": rationale, + "result_json": result.model_dump_json(), + "created_at": self.store.now(), + }) + + async def asave_trace_result(self, *args, **kwargs): + return await self.store.to_thread(self.save_trace_result, *args, **kwargs) + + def save_session_result(self, run_id: str, session_id: str, agent_id: str | None, result: SessionJudgeResult): + judge_name = getattr(result, "judge_name", None) or "session_metrics" + judge_type = (getattr(result, "judge_type", None) or "SESSION").upper() + rationale = getattr(result, "rationale", None) or "" + with self.store.connect() as conn: + conn.cursor().execute(f""" + insert into {self.store.t('EVALUATION_RESULT')} + (RESULT_ID, RUN_ID, SESSION_ID, AGENT_ID, JUDGE_NAME, JUDGE_TYPE, + INFERRED_CSI_SCORE, RESOLUTION, CONVERSATION_PRECISION, RATIONALE, + RESULT_JSON, CREATED_AT) + values (:result_id, :run_id, :session_id, :agent_id, :judge_name, :judge_type, + :csi, :resolution, :precision, :rationale, :result_json, :created_at) + """, { + "result_id": str(uuid.uuid4()), + "run_id": run_id, + "session_id": session_id, + "agent_id": agent_id, + "judge_name": judge_name, + "judge_type": judge_type, + "csi": getattr(result, "inferredCsiScore", None), + "resolution": getattr(result, "resolution", None), + "precision": getattr(result, "conversationPrecision", None), + "rationale": rationale, + "result_json": result.model_dump_json(), + "created_at": self.store.now(), + }) + + async def asave_session_result(self, *args, **kwargs): + return await self.store.to_thread(self.save_session_result, *args, **kwargs) + + def mark_run_status(self, run_id: str, status: RunStatus, error: str | None = None): + with self.store.connect() as conn: + conn.cursor().execute(f""" + update {self.store.t('EVALUATION_RUN')} + set STATUS = :status, + ERROR_MESSAGE = :error, + UPDATED_AT = :updated_at + where RUN_ID = :run_id + """, { + "status": status.value, + "error": error, + "updated_at": self.store.now(), + "run_id": run_id, + }) + + async def amark_run_status(self, *args, **kwargs): + return await self.store.to_thread(self.mark_run_status, *args, **kwargs) + + def summarize_run(self, run_id: str) -> dict: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id), + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'COMPLETED'), + (select count(*) from {self.store.t('EVALUATION_ITEM')} where RUN_ID = :run_id and STATUS = 'FAILED'), + (select count(*) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE'), + (select avg(JUDGE_SCORE) from {self.store.t('EVALUATION_RESULT')} where RUN_ID = :run_id and JUDGE_TYPE = 'TRACE') + from dual + """, {"run_id": run_id}) + r = cur.fetchone() + return { + "run_id": run_id, + "total_items": int(r[0] or 0), + "completed_items": int(r[1] or 0), + "failed_items": int(r[2] or 0), + "evaluations": int(r[3] or 0), + "avg_score": float(r[4]) if r[4] is not None else None, + } + + async def asummarize_run(self, *args, **kwargs): + return await self.store.to_thread(self.summarize_run, *args, **kwargs) + + def get_run_progress(self, run_id: str, event_limit: int = 20) -> dict: + summary = self.summarize_run(run_id) + total = summary["total_items"] or 0 + done = summary["completed_items"] + summary["failed_items"] + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select STAGE, MESSAGE, DETAILS_JSON, CREATED_AT + from {self.store.t('EVALUATION_PROGRESS_EVENT')} + where RUN_ID = :run_id + order by CREATED_AT desc + ) where rownum <= :max_rows + """, {"run_id": run_id, "max_rows": event_limit}) + events = [ + { + "stage": s, + "message": m, + "details": _json_loads(d.read() if hasattr(d, "read") else d, {}), + "created_at": str(c), + } + for s, m, d, c in cur.fetchall() + ] + return { + **summary, + "done_items": done, + "percent_complete": round((done / total) * 100, 2) if total else 0.0, + "events": events, + } + + async def aget_run_progress(self, *args, **kwargs): + return await self.store.to_thread(self.get_run_progress, *args, **kwargs) + + def list_runs(self, limit: int = 20): + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select * from ( + select RUN_ID, AGENT_ID, PERIOD_START, PERIOD_END, SOURCE, STATUS, + TOTAL_ITEMS, PROCESSED_ITEMS, FAILED_ITEMS, CREATED_AT, UPDATED_AT + from {self.store.t('EVALUATION_RUN')} + order by CREATED_AT desc + ) where rownum <= :max_rows + """, {"max_rows": limit}) + return [ + { + "run_id": r[0], + "agent_id": r[1], + "period_start": str(r[2]), + "period_end": str(r[3]), + "source": r[4], + "status": r[5], + "total_items": int(r[6] or 0), + "processed_items": int(r[7] or 0), + "failed_items": int(r[8] or 0), + "created_at": str(r[9]), + "updated_at": str(r[10]), + } + for r in cur.fetchall() + ] + + async def alist_runs(self, *args, **kwargs): + return await self.store.to_thread(self.list_runs, *args, **kwargs) + + def list_results(self, run_id: str, limit: int = 100) -> list[dict]: + with self.store.connect() as conn: + cur = conn.cursor() + cur.execute(f""" + select JUDGE_NAME, JUDGE_TYPE, SCORE, JUDGE_SCORE, ACCURACY_SCORE, + ALUCINATION_SCORE, INFERRED_CSI_SCORE, RESOLUTION, + CONVERSATION_PRECISION, RATIONALE, TRACE_ID, SESSION_ID, CREATED_AT + from {self.store.t('EVALUATION_RESULT')} + where RUN_ID = :run_id + order by CREATED_AT desc + """, {"run_id": run_id}) + return [ + { + "judge_name": r[0], + "judge_type": r[1], + "score": r[2], + "judge_score": r[3], + "accuracy_score": r[4], + "alucination_score": r[5], + "inferred_csi_score": r[6], + "resolution": r[7], + "conversation_precision": r[8], + "rationale": r[9], + "trace_id": r[10], + "session_id": r[11], + "created_at": str(r[12]), + } + for r in cur.fetchall()[:limit] + ] + + async def alist_results(self, *args, **kwargs): + return await self.store.to_thread(self.list_results, *args, **kwargs) diff --git a/evaluator/prompts/__pycache__/loader.cpython-313.pyc b/evaluator/prompts/__pycache__/loader.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d3a7bd05d510aec53b85bfe9e53cce0d60da548d GIT binary patch literal 862 zcmaJ}9UJ4Zt>aG5R{)|8m%9x%!c@ZIacV?Sa>7fty?f1TU@0<5#W{}N}fwG@$ar+4Y z_^yM*j0->p1qF_v0u@q%HG;&^*eDrjsJ~e<*DSOYm#SDZux(+gVdT?4ba9?Yn}%T| zs0gBvud6u2nGiwP;G08vod;?4nE+(aQ{V_{nv1Rcf7XzjhbkTX-FkHD`a8F_-aQ7?m%d6Ox{H|XJ5i{RDp@viYW%A5_pGFzkyLMK0hdIo8=so0WMm$sU6l+!x% zni2KoC^G7Mf;|^9Q)1mmh!IMY3(Q-nCzhrXm82~#p9;#gl%q`YVwE%G#Z3&j2xu5> zMSLsRp}VxW8tsN!n-^(=g~F>NZL!@5cRcotHak>Ac*V-UoOV1EFKR7@g?3lk8hRHz zPN-INhrF!ZKJ+sahuMSdTmLLGyZ@kXj~}`R?(y7*#dnMEZ=cx str: + p = settings.path(path) + data = yaml.safe_load(p.read_text()) or {} + if key not in data: + raise KeyError(f"Prompt key {key!r} not found in {p}") + return str(data[key]) diff --git a/evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc b/evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0238fa165db486199e98b7222fb744553624d371 GIT binary patch literal 2541 zcmb6bOKclObY^$`Nn*!ooaWyoBtgZc*exy4FQ}wRn@=O+P2`q@GOyFR)_^Q_$G!~gKCng=eX`Jr<{?N~ zB)+T^^Rw0o)y!rbs>@(nf5(~28#Jer4L>MrVvXG(;GaQ#w*CYM1E>#7&sN#VvePO7 zkeA9Ebq_)_I9aaYZW3vXP`1_@cmB*%hWcp^LC^~-;A=5wOOLkP`cUC!YQImtm#ZkEDPiY2b0?E!Dxe6xNo%jx+G~9*@ zuRt~Hcb5WccSU1ol^fzB75Wh6y##-iUKxdbt?xcGh#KJj{hyQ^lEa7aAPS<%Wq4=d z?S;4T1#}R(HsMBkl?b9x^|tc6n1LvI!2|dx+c+)uq6~gR%-~q#B#T;^Ox`ecWuCX7 zLV4%JBo?rFTaKogWU7lg^DV1+M`r=eT3pmko3&Q7R84b0a>Y|*9aVbX(G1gVbdR;c zI~Zm_a89MV;*dPJFcx*ou|I51s7pp-iMOkka0B4OLYk+R$t2aWbH*wpG7=M+U@b5| zh3Vo@Ce2yt719P)r(+SqJCb#2s;RhkXimwOXMyLQvY?xhVx(EL%JO_F3;Ie8k?I=J zZI@W@7R8+=td6+ZsCk`*D_MJ-Ca6s%S&!n&+4j;E)^LWdnA%X@;3W%Jz2+=xGiiy# z1}_;!v(h?~Z2gKI3o?o7`3&#pztZ`DmNB(1-vcM{=T&dcTx1iz#yxp{w|90i>_u2`*h; z!WV?ifYefdJ@`jGYUwCN2k%7(d0=OY@w0d5O3|r%(W%1B?1Qd>)xiCI1Gi>x&K6>) zzd!fIY++xj)RtP6Af-}tpcozalKvDu@VI~fH%CiHhKfgq?oR)B>4!_ju2iulwcUVP zk{HYtqf~;r*fY7egFBTo2r@&}2*^F^Ge}&lE%saKJy9@G%c6 z3Lia+!s-=e(Mmgcj)xTG14qr5IT1z47=+q= 400: + # Don't fail the run because score publishing is supplementary. + return {'ok': False, 'status': resp.status_code, 'body': resp.text} + return {'ok': True} diff --git a/k8s/cronjob.yaml b/k8s/cronjob.yaml new file mode 100644 index 0000000..c317248 --- /dev/null +++ b/k8s/cronjob.yaml @@ -0,0 +1,20 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: agent-framework-evaluator +spec: + schedule: "0 2 * * *" + suspend: true + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + containers: + - name: evaluator + image: agent-framework-evaluator:latest + command: ["python", "-m", "evaluator.cli", "run-agents", "--source", "langfuse"] + envFrom: + - secretRef: + name: agent-framework-evaluator-env diff --git a/output/.DS_Store b/output/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0=1.0.1", + "pydantic>=2.7.0", + "pydantic-settings>=2.2.1", + "oracledb>=2.4.0", + "httpx>=0.27.0", + "pyyaml>=6.0.1", + "typer>=0.12.3", + "click>=8.1.7", + "rich>=13.7.0", + "fastapi>=0.111.0", + "uvicorn>=0.30.0" +] + +[project.scripts] +af-evaluator = "evaluator.cli:app" + +[tool.setuptools.packages.find] +where = ["."] +include = ["evaluator*"]