diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 5cf0f44..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.env.example b/.env.example old mode 100644 new mode 100755 index cca14aa..d90bc55 --- a/.env.example +++ b/.env.example @@ -44,6 +44,8 @@ AGENTS_CONFIG_PATH=configs/judge/agents.yaml TRACE_PROMPT_PATH=configs/judge/trace_metrics.yaml SESSION_PROMPT_PATH=configs/judge/session_metrics.yaml OUTPUT_DIR=output +# Available placeholders: {agent_id}, {run_id}, {timestamp} (YYYYMMDD_HHMMSS) +OUTPUT_FILENAME_PATTERN=AGENTE_{agent_id}_LLM_JUDGE_{timestamp}.TXT.GZ BATCH_SIZE=50 MAX_ATTEMPTS=3 CONTEXT_TURN_LIMIT=6 @@ -58,4 +60,3 @@ GOOGLE_APPLICATION_CREDENTIALS=configs/GCP_ACCESS_KEY.json # Langfuse credentials are declared in one external YAML file. # In production, mount it from a Kubernetes Secret. LANGFUSE_CREDENTIALS_CONFIG_PATH=configs/langfuse/langfuse_credentials.yaml - diff --git a/Dockerfile b/Dockerfile old mode 100644 new mode 100755 diff --git a/Documentation/llm-as-a-judge.pdf b/Documentation/llm-as-a-judge.pdf old mode 100644 new mode 100755 diff --git a/README.en-US.md b/README.en-US.md old mode 100644 new mode 100755 index 00511a4..1fc63e9 --- a/README.en-US.md +++ b/README.en-US.md @@ -688,7 +688,16 @@ GOOGLE_APPLICATION_CREDENTIALS=/path/GCP_ACCESS_KEY.json ## 12. Output file -Pattern: +The name is configured directly in `.env`: + +```env +OUTPUT_FILENAME_PATTERN=AGENTE_{agent_id}_LLM_JUDGE_{timestamp}.TXT.GZ +``` + +Available placeholders are `{agent_id}`, `{run_id}`, and `{timestamp}`. The +`timestamp` uses the `YYYYMMDD_HHMMSS` format in the São Paulo time zone. + +Resulting default pattern: ```text AGENTE_{agentId}_LLM_JUDGE_{YYYYMMDD}_{HHMMSS}.TXT.GZ @@ -1463,6 +1472,7 @@ python -m evaluator.cli progress --events 100 | `JUDGE_GCS_BUCKET` | bucket override | | `GOOGLE_APPLICATION_CREDENTIALS` | GCP credentials | | `OUTPUT_DIR` | temporary export directory | +| `OUTPUT_FILENAME_PATTERN` | configurable output filename template | --- diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 3c798cc..f54c548 --- a/README.md +++ b/README.md @@ -688,7 +688,16 @@ GOOGLE_APPLICATION_CREDENTIALS=/path/GCP_ACCESS_KEY.json ## 12. Arquivo de saída -Padrão: +O nome é configurado diretamente no `.env`: + +```env +OUTPUT_FILENAME_PATTERN=AGENTE_{agent_id}_LLM_JUDGE_{timestamp}.TXT.GZ +``` + +Os placeholders disponíveis são `{agent_id}`, `{run_id}` e `{timestamp}`. O +`timestamp` é gerado no formato `YYYYMMDD_HHMMSS`, no fuso de São Paulo. + +Padrão resultante: ```text AGENTE_{agentId}_LLM_JUDGE_{YYYYMMDD}_{HHMMSS}.TXT.GZ @@ -1478,6 +1487,7 @@ Abra `http://localhost:8080/ui`. Em Kubernetes, prefira `kubectl port-forward sv | `JUDGE_GCS_BUCKET` | override de bucket | | `GOOGLE_APPLICATION_CREDENTIALS` | credencial GCP | | `OUTPUT_DIR` | diretório temporário de export | +| `OUTPUT_FILENAME_PATTERN` | template configurável do nome do arquivo de saída | --- diff --git a/configs/.DS_Store b/configs/.DS_Store deleted file mode 100644 index 08521bc..0000000 Binary files a/configs/.DS_Store and /dev/null differ diff --git a/configs/identity.yaml b/configs/identity.yaml old mode 100644 new mode 100755 diff --git a/configs/judge/agents.yaml b/configs/judge/agents.yaml old mode 100644 new mode 100755 diff --git a/configs/judge/session_metrics.yaml b/configs/judge/session_metrics.yaml old mode 100644 new mode 100755 diff --git a/configs/judge/storage.yaml b/configs/judge/storage.yaml old mode 100644 new mode 100755 diff --git a/configs/judge/trace_metrics.yaml b/configs/judge/trace_metrics.yaml old mode 100644 new mode 100755 diff --git a/configs/langfuse/langfuse_credentials.example.yaml b/configs/langfuse/langfuse_credentials.example.yaml old mode 100644 new mode 100755 diff --git a/configs/llm_profiles/llm_profiles.yaml b/configs/llm_profiles/llm_profiles.yaml old mode 100644 new mode 100755 diff --git a/deploy/.DS_Store b/deploy/.DS_Store deleted file mode 100644 index 01a15a8..0000000 Binary files a/deploy/.DS_Store and /dev/null differ diff --git a/deploy/README.md b/deploy/README.md old mode 100644 new mode 100755 diff --git a/deploy/env/evaluator.env.example b/deploy/env/evaluator.env.example old mode 100644 new mode 100755 index 4d80ed3..400bbd3 --- a/deploy/env/evaluator.env.example +++ b/deploy/env/evaluator.env.example @@ -17,6 +17,7 @@ ENABLE_GCS_UPLOAD=true JUDGE_GCS_BUCKET= STORAGE_CONFIG_PATH=configs/judge/storage.yaml GOOGLE_APPLICATION_CREDENTIALS=/var/secrets/google/GCP_ACCESS_KEY.json +OUTPUT_FILENAME_PATTERN=AGENTE_{agent_id}_LLM_JUDGE_{timestamp}.TXT.GZ # Langfuse per-agent credentials are supplied by the mounted YAML file, not env names. LANGFUSE_CREDENTIALS_CONFIG_PATH=/var/secrets/langfuse/langfuse_credentials.yaml diff --git a/deploy/k8s/.DS_Store b/deploy/k8s/.DS_Store deleted file mode 100644 index 707cc98..0000000 Binary files a/deploy/k8s/.DS_Store and /dev/null differ diff --git a/deploy/k8s/base/configmap.yaml b/deploy/k8s/base/configmap.yaml old mode 100644 new mode 100755 index 22edd21..f677fd4 --- a/deploy/k8s/base/configmap.yaml +++ b/deploy/k8s/base/configmap.yaml @@ -20,6 +20,7 @@ data: BATCH_SIZE: "50" MAX_ATTEMPTS: "3" ENABLE_GCS_UPLOAD: "true" + OUTPUT_FILENAME_PATTERN: "AGENTE_{agent_id}_LLM_JUDGE_{timestamp}.TXT.GZ" STORAGE_CONFIG_PATH: "configs/judge/storage.yaml" ADB_TABLE_PREFIX: "AGENTFW" ADB_WALLET_LOCATION: "/opt/oracle/wallet" diff --git a/deploy/k8s/base/cronjob.yaml b/deploy/k8s/base/cronjob.yaml old mode 100644 new mode 100755 diff --git a/deploy/k8s/base/init-db-job.yaml b/deploy/k8s/base/init-db-job.yaml old mode 100644 new mode 100755 diff --git a/deploy/k8s/base/kustomization.yaml b/deploy/k8s/base/kustomization.yaml old mode 100644 new mode 100755 diff --git a/deploy/k8s/base/monitor-deployment.yaml b/deploy/k8s/base/monitor-deployment.yaml old mode 100644 new mode 100755 diff --git a/deploy/k8s/base/monitor-service.yaml b/deploy/k8s/base/monitor-service.yaml old mode 100644 new mode 100755 diff --git a/deploy/k8s/base/namespace.yaml b/deploy/k8s/base/namespace.yaml old mode 100644 new mode 100755 diff --git a/deploy/k8s/base/secret-template.yaml b/deploy/k8s/base/secret-template.yaml old mode 100644 new mode 100755 diff --git a/deploy/k8s/base/serviceaccount.yaml b/deploy/k8s/base/serviceaccount.yaml old mode 100644 new mode 100755 diff --git a/deploy/k8s/overlays/dev/kustomization.yaml b/deploy/k8s/overlays/dev/kustomization.yaml old mode 100644 new mode 100755 diff --git a/deploy/k8s/overlays/hml/kustomization.yaml b/deploy/k8s/overlays/hml/kustomization.yaml old mode 100644 new mode 100755 diff --git a/deploy/k8s/overlays/prod/kustomization.yaml b/deploy/k8s/overlays/prod/kustomization.yaml old mode 100644 new mode 100755 diff --git a/deploy/langfuse/langfuse_credentials.example.yaml b/deploy/langfuse/langfuse_credentials.example.yaml old mode 100644 new mode 100755 diff --git a/deploy/scripts/build-push.sh b/deploy/scripts/build-push.sh old mode 100644 new mode 100755 diff --git a/deploy/scripts/common.sh b/deploy/scripts/common.sh old mode 100644 new mode 100755 diff --git a/deploy/scripts/create-secrets.sh b/deploy/scripts/create-secrets.sh old mode 100644 new mode 100755 diff --git a/deploy/scripts/deploy.sh b/deploy/scripts/deploy.sh old mode 100644 new mode 100755 diff --git a/deploy/scripts/run-now.sh b/deploy/scripts/run-now.sh old mode 100644 new mode 100755 diff --git a/deploy/scripts/status.sh b/deploy/scripts/status.sh old mode 100644 new mode 100755 diff --git a/deploy/scripts/undeploy.sh b/deploy/scripts/undeploy.sh old mode 100644 new mode 100755 diff --git a/evaluator/.DS_Store b/evaluator/.DS_Store deleted file mode 100644 index fe416d7..0000000 Binary files a/evaluator/.DS_Store and /dev/null differ diff --git a/evaluator/__init__.py b/evaluator/__init__.py old mode 100644 new mode 100755 diff --git a/evaluator/__pycache__/__init__.cpython-313.pyc b/evaluator/__pycache__/__init__.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/__pycache__/cli.cpython-313.pyc b/evaluator/__pycache__/cli.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/__pycache__/engine.cpython-313.pyc b/evaluator/__pycache__/engine.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/__pycache__/time_utils.cpython-313.pyc b/evaluator/__pycache__/time_utils.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/analytics/__init__.py b/evaluator/analytics/__init__.py old mode 100644 new mode 100755 diff --git a/evaluator/analytics/__pycache__/__init__.cpython-313.pyc b/evaluator/analytics/__pycache__/__init__.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/analytics/__pycache__/vloop.cpython-313.pyc b/evaluator/analytics/__pycache__/vloop.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/analytics/vloop.py b/evaluator/analytics/vloop.py old mode 100644 new mode 100755 diff --git a/evaluator/cli.py b/evaluator/cli.py old mode 100644 new mode 100755 diff --git a/evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc b/evaluator/collectors/__pycache__/agent_framework.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/collectors/__pycache__/base.cpython-313.pyc b/evaluator/collectors/__pycache__/base.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/collectors/__pycache__/langfuse.cpython-313.pyc b/evaluator/collectors/__pycache__/langfuse.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/collectors/__pycache__/langfuse_credentials.cpython-313.pyc b/evaluator/collectors/__pycache__/langfuse_credentials.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/collectors/__pycache__/mock.cpython-313.pyc b/evaluator/collectors/__pycache__/mock.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/collectors/agent_framework.py b/evaluator/collectors/agent_framework.py old mode 100644 new mode 100755 diff --git a/evaluator/collectors/base.py b/evaluator/collectors/base.py old mode 100644 new mode 100755 diff --git a/evaluator/collectors/langfuse.py b/evaluator/collectors/langfuse.py old mode 100644 new mode 100755 diff --git a/evaluator/collectors/langfuse_credentials.py b/evaluator/collectors/langfuse_credentials.py old mode 100644 new mode 100755 diff --git a/evaluator/collectors/mock.py b/evaluator/collectors/mock.py old mode 100644 new mode 100755 diff --git a/evaluator/config/__pycache__/agents.cpython-313.pyc b/evaluator/config/__pycache__/agents.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/config/__pycache__/settings.cpython-313.pyc b/evaluator/config/__pycache__/settings.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/config/agents.py b/evaluator/config/agents.py old mode 100644 new mode 100755 diff --git a/evaluator/config/settings.py b/evaluator/config/settings.py old mode 100644 new mode 100755 index 060e8d2..eef2da1 --- a/evaluator/config/settings.py +++ b/evaluator/config/settings.py @@ -71,6 +71,10 @@ class Settings(BaseSettings): trace_prompt_path: str = Field(default="configs/judge/trace_metrics.yaml", validation_alias="TRACE_PROMPT_PATH") session_prompt_path: str = Field(default="configs/judge/session_metrics.yaml", validation_alias="SESSION_PROMPT_PATH") output_dir: str = Field(default="output", validation_alias="OUTPUT_DIR") + output_filename_pattern: str = Field( + default="AGENTE_{agent_id}_LLM_JUDGE_{timestamp}.TXT.GZ", + validation_alias="OUTPUT_FILENAME_PATTERN", + ) batch_size: int = Field(default=50, validation_alias="BATCH_SIZE") max_attempts: int = Field(default=3, validation_alias="MAX_ATTEMPTS") context_turn_limit: int = Field(default=6, validation_alias="CONTEXT_TURN_LIMIT") diff --git a/evaluator/core/__pycache__/models.cpython-313.pyc b/evaluator/core/__pycache__/models.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/core/models.py b/evaluator/core/models.py old mode 100644 new mode 100755 diff --git a/evaluator/engine.py b/evaluator/engine.py old mode 100644 new mode 100755 diff --git a/evaluator/identity/__pycache__/resolver.cpython-313.pyc b/evaluator/identity/__pycache__/resolver.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/identity/resolver.py b/evaluator/identity/resolver.py old mode 100644 new mode 100755 diff --git a/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc b/evaluator/judges/__pycache__/llm_judge.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/judges/llm_judge.py b/evaluator/judges/llm_judge.py old mode 100644 new mode 100755 diff --git a/evaluator/llm/__pycache__/client.cpython-313.pyc b/evaluator/llm/__pycache__/client.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc b/evaluator/llm/__pycache__/profile_resolver.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/llm/client.py b/evaluator/llm/client.py old mode 100644 new mode 100755 diff --git a/evaluator/llm/profile_resolver.py b/evaluator/llm/profile_resolver.py old mode 100644 new mode 100755 diff --git a/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc b/evaluator/output/__pycache__/legacy_exporter.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/output/legacy_exporter.py b/evaluator/output/legacy_exporter.py old mode 100644 new mode 100755 index 0fc268e..3eee0ee --- a/evaluator/output/legacy_exporter.py +++ b/evaluator/output/legacy_exporter.py @@ -106,11 +106,36 @@ def _session_loop_map(rows) -> dict[str, int]: return {sid: max_consecutive_intent_streak(items) for sid, items in grouped.items()} +def build_output_filename( + agent_id: str, + run_id: str, + now_brt: datetime, + pattern: str | None = None, +) -> str: + """Build the export filename from the environment-configurable template.""" + template = pattern or settings.output_filename_pattern + try: + filename = template.format( + agent_id=agent_id, + run_id=run_id, + timestamp=now_brt.strftime("%Y%m%d_%H%M%S"), + ) + except KeyError as exc: + raise ValueError( + f"Unknown placeholder {exc} in OUTPUT_FILENAME_PATTERN. " + "Supported placeholders: {agent_id}, {run_id}, {timestamp}." + ) from exc + + if not filename or Path(filename).name != filename or filename in {".", ".."}: + raise ValueError("OUTPUT_FILENAME_PATTERN must produce a filename without directories") + return filename + + def export_legacy_txt_gz(repo: EvaluationRepository, run_id: str, agent_id: str) -> Path: output_dir = settings.path(settings.output_dir) output_dir.mkdir(parents=True, exist_ok=True) now_brt = datetime.now(ZoneInfo("America/Sao_Paulo")) - path = output_dir / f"AGENTE_{agent_id}_LLM_JUDGE_{now_brt.strftime('%Y%m%d_%H%M%S')}.TXT.GZ" + path = output_dir / build_output_filename(agent_id, run_id, now_brt) with repo.store.connect() as conn: cur = conn.cursor() diff --git a/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc b/evaluator/persistence/__pycache__/oracle_store.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/persistence/__pycache__/repository.cpython-313.pyc b/evaluator/persistence/__pycache__/repository.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/persistence/oracle_store.py b/evaluator/persistence/oracle_store.py old mode 100644 new mode 100755 diff --git a/evaluator/persistence/repository.py b/evaluator/persistence/repository.py old mode 100644 new mode 100755 diff --git a/evaluator/prompts/__pycache__/loader.cpython-313.pyc b/evaluator/prompts/__pycache__/loader.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/prompts/loader.py b/evaluator/prompts/loader.py old mode 100644 new mode 100755 diff --git a/evaluator/publishers/__pycache__/gcs_export.cpython-313.pyc b/evaluator/publishers/__pycache__/gcs_export.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/publishers/__pycache__/langfuse_job_audit.cpython-313.pyc b/evaluator/publishers/__pycache__/langfuse_job_audit.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc b/evaluator/publishers/__pycache__/langfuse_scores.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator/publishers/gcs_export.py b/evaluator/publishers/gcs_export.py old mode 100644 new mode 100755 diff --git a/evaluator/publishers/langfuse_job_audit.py b/evaluator/publishers/langfuse_job_audit.py old mode 100644 new mode 100755 diff --git a/evaluator/publishers/langfuse_scores.py b/evaluator/publishers/langfuse_scores.py old mode 100644 new mode 100755 diff --git a/evaluator/time_utils.py b/evaluator/time_utils.py old mode 100644 new mode 100755 diff --git a/evaluator_monitor/__init__.py b/evaluator_monitor/__init__.py old mode 100644 new mode 100755 diff --git a/evaluator_monitor/__pycache__/__init__.cpython-313.pyc b/evaluator_monitor/__pycache__/__init__.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator_monitor/__pycache__/app.cpython-313.pyc b/evaluator_monitor/__pycache__/app.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/evaluator_monitor/app.py b/evaluator_monitor/app.py old mode 100644 new mode 100755 diff --git a/pyproject.toml b/pyproject.toml old mode 100644 new mode 100755 diff --git a/tests/__init__.py b/tests/__init__.py old mode 100644 new mode 100755 diff --git a/tests/__pycache__/__init__.cpython-313.pyc b/tests/__pycache__/__init__.cpython-313.pyc old mode 100644 new mode 100755 diff --git a/tests/__pycache__/test_full_tim_compliance.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_full_tim_compliance.cpython-313-pytest-9.0.2.pyc old mode 100644 new mode 100755 diff --git a/tests/__pycache__/test_monitor_and_run_id.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_monitor_and_run_id.cpython-313-pytest-9.0.2.pyc old mode 100644 new mode 100755 diff --git a/tests/__pycache__/test_no_product_hardcodes.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_no_product_hardcodes.cpython-313-pytest-9.0.2.pyc old mode 100644 new mode 100755 diff --git a/tests/__pycache__/test_tim_spec.cpython-313-pytest-9.0.2.pyc b/tests/__pycache__/test_tim_spec.cpython-313-pytest-9.0.2.pyc old mode 100644 new mode 100755 diff --git a/tests/test_full_tim_compliance.py b/tests/test_full_tim_compliance.py old mode 100644 new mode 100755 diff --git a/tests/test_monitor_and_run_id.py b/tests/test_monitor_and_run_id.py old mode 100644 new mode 100755 diff --git a/tests/test_no_product_hardcodes.py b/tests/test_no_product_hardcodes.py old mode 100644 new mode 100755 diff --git a/tests/test_tim_spec.py b/tests/test_tim_spec.py old mode 100644 new mode 100755 index 5dac84d..d1b153e --- a/tests/test_tim_spec.py +++ b/tests/test_tim_spec.py @@ -10,7 +10,7 @@ from evaluator.config.agents import load_agents from evaluator.core.models import ConversationRecord from evaluator.engine import EvaluationEngine from evaluator.judges.llm_judge import TIMStyleLLMJudge -from evaluator.output.legacy_exporter import extract_prompt_length +from evaluator.output.legacy_exporter import build_output_filename, extract_prompt_length from evaluator.time_utils import brt_day_window, to_utc_z @@ -25,6 +25,20 @@ class FakeLLM: class TIMSpecTests(unittest.TestCase): + def test_output_filename_pattern_is_configurable(self): + now_brt = datetime(2026, 9, 9, 18, 45, 12) + filename = build_output_filename( + "billing", + "run-123", + now_brt, + pattern="judge_{agent_id}_{run_id}_{timestamp}.csv.gz", + ) + self.assertEqual(filename, "judge_billing_run-123_20260909_184512.csv.gz") + + def test_output_filename_pattern_rejects_directories(self): + with self.assertRaisesRegex(ValueError, "without directories"): + build_output_filename("billing", "run-123", datetime.now(), pattern="../{run_id}.gz") + def test_brt_day_converts_to_expected_utc(self): start, end = brt_day_window(date(2026, 8, 12)) self.assertEqual(to_utc_z(start), "2026-08-12T03:00:00Z")