first commit

tests/test_full_tim_compliance.py

@@ -0,0 +1,279 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import tempfile
+import unittest
+from datetime import datetime
+from pathlib import Path
+from unittest.mock import patch
+
+from evaluator.collectors.langfuse import LangfuseCollector
+from evaluator.collectors.langfuse_credentials import resolve_langfuse_credentials
+from evaluator.config.agents import load_agents
+from evaluator.output.legacy_exporter import extract_prompt_length
+
+
+class _Response:
+    def __init__(self, payload, status_code=200):
+        self._payload = payload
+        self.status_code = status_code
+        self.text = ""
+    def json(self):
+        return self._payload
+
+
+class _Client:
+    def __init__(self, pages):
+        self.pages = pages
+        self.calls = []
+    async def get(self, path, params=None, auth=None):
+        self.calls.append((path, dict(params or {})))
+        page = int((params or {}).get("page", 1))
+        return _Response(self.pages.get((path, page), {"data": []}))
+
+
+class FullTIMComplianceTests(unittest.TestCase):
+    def test_per_agent_langfuse_credentials_are_data_driven(self):
+        from evaluator.config.settings import settings
+        with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f:
+            f.write(
+                "langfuse:\n"
+                "  agents:\n"
+                "    - agentId: arbitrary-new-agent\n"
+                "      host: https://lf-arbitrary.example\n"
+                "      publicKey: pk-arbitrary\n"
+                "      secretKey: sk-arbitrary\n"
+            )
+            path = f.name
+        old = settings.langfuse_credentials_config_path
+        try:
+            settings.langfuse_credentials_config_path = path
+            c = resolve_langfuse_credentials("arbitrary-new-agent")
+            self.assertEqual(c.public_key, "pk-arbitrary")
+            self.assertEqual(c.secret_key, "sk-arbitrary")
+            self.assertEqual(c.host, "https://lf-arbitrary.example")
+            self.assertIn("arbitrary-new-agent", c.source)
+        finally:
+            settings.langfuse_credentials_config_path = old
+            Path(path).unlink(missing_ok=True)
+
+    def test_agents_yaml_accepts_document_camel_case(self):
+        with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f:
+            f.write("agents:\n  - agentId: billing\n    daysBack: 2\n    percentage: 0.5\n    gcs_prefix: agnt_ai_contas\n")
+            path = f.name
+        try:
+            agents = load_agents(path)
+            self.assertEqual(agents[0].agent_id, "billing")
+            self.assertEqual(agents[0].days_back, 2)
+            self.assertEqual(agents[0].percentage, 0.5)
+        finally:
+            Path(path).unlink(missing_ok=True)
+
+    def test_pagination_reads_all_pages_without_n_plus_one(self):
+        c = LangfuseCollector("billing")
+        # PAGE_SIZE is changed only inside this test to keep fixtures small.
+        c.PAGE_SIZE = 2
+        client = _Client({
+            ("/api/public/traces", 1): {"data": [{"id": "t1"}, {"id": "t2"}], "meta": {"totalPages": 2}},
+            ("/api/public/traces", 2): {"data": [{"id": "t3"}], "meta": {"totalPages": 2}},
+        })
+        from evaluator.collectors.langfuse_credentials import LangfuseCredentials
+        creds = LangfuseCredentials("http://x", "pk", "sk", "test")
+        rows = asyncio.run(c._fetch_all_pages(client, "/api/public/traces", {}, ("traces",), creds))
+        self.assertEqual([x["id"] for x in rows], ["t1", "t2", "t3"])
+        self.assertEqual(len(client.calls), 2)
+        self.assertFalse(any("/api/public/traces/t" in path for path, _ in client.calls))
+
+    def test_trace_metadata_precedes_session_metadata_and_session_falls_back(self):
+        c = LangfuseCollector("billing")
+        trace = {
+            "id": "t1", "sessionId": "s1", "input": "abc", "output": "ok",
+            "metadata": {"ura_call_id": "trace-call", "channelId": "trace-channel", "message_id": "trace-message"},
+        }
+        session = {"id": "s1", "metadata": {"ura_call_id": "session-call", "channelId": "session-channel", "message_id": "session-message"}}
+        record = c._to_record(trace, [], session, "billing")
+        self.assertEqual(record.metadata["ura_call_id"], "trace-call")
+        self.assertEqual(record.channel, "trace-channel")
+        self.assertEqual(record.message_id, "trace-message")
+        self.assertEqual(record.raw["prompt_length"], 3)
+
+        trace["metadata"] = {}
+        record2 = c._to_record(trace, [], session, "billing")
+        self.assertEqual(record2.metadata["ura_call_id"], "session-call")
+        self.assertEqual(record2.channel, "session-channel")
+        self.assertEqual(record2.message_id, "session-message")
+
+    def test_export_prompt_length_prefers_raw_trace_count(self):
+        raw = {"input_text": "normalized", "raw": {"prompt_length": 123}}
+        self.assertEqual(extract_prompt_length(raw), 123)
+
+
+    def test_alias_filter_is_enforced_with_yaml_per_agent_credentials(self):
+        from evaluator.collectors.langfuse_credentials import LangfuseCredentials
+        from evaluator.config.settings import settings
+
+        class Collector(LangfuseCollector):
+            async def _fetch_sessions(self, client, start, end, creds):
+                return [{"id": "s-agent-a"}, {"id": "s-agent-b"}]
+
+            async def _fetch_traces(self, client, start, end, creds):
+                return [
+                    {
+                        "id": "t-agent-a",
+                        "sessionId": "s-agent-a",
+                        "name": "agent-a",
+                        "input": "hello a",
+                        "output": "answer a",
+                    },
+                    {
+                        "id": "t-agent-b",
+                        "sessionId": "s-agent-b",
+                        "name": "agent-b",
+                        "input": "hello b",
+                        "output": "answer b",
+                    },
+                ]
+
+            async def _fetch_observations(self, client, start, end, creds):
+                return []
+
+        class NoopAsyncClient:
+            def __init__(self, *args, **kwargs):
+                pass
+            async def __aenter__(self):
+                return self
+            async def __aexit__(self, exc_type, exc, tb):
+                return False
+
+        collector = Collector("agent-a")
+        creds = LangfuseCredentials(
+            "https://langfuse-agent-a.example", "pk", "sk",
+            "/mounted/langfuse_credentials.yaml#agent-a",
+        )
+        old_enable = settings.enable_langfuse
+        try:
+            settings.enable_langfuse = True
+            with patch("evaluator.collectors.langfuse.resolve_langfuse_credentials", return_value=creds), \
+                 patch("evaluator.collectors.langfuse.httpx.AsyncClient", NoopAsyncClient):
+                records, sampled, stats = asyncio.run(collector.collect_sampled(
+                    datetime(2026, 8, 13, 0, 0),
+                    datetime(2026, 8, 13, 23, 59, 59),
+                    agent_aliases={"agent-a"},
+                    percentage=1.0,
+                    seed="agent-a:2026-08-13",
+                ))
+            self.assertEqual(sampled, ["s-agent-a"])
+            self.assertEqual([r.trace_id for r in records], ["t-agent-a"])
+            self.assertEqual(stats["selected_traces"], 1)
+            self.assertIn("#agent-a", stats["credential_source"])
+        finally:
+            settings.enable_langfuse = old_enable
+
+    def test_evaluator_remains_cronjob_only_and_monitor_is_separate(self):
+        root = Path(__file__).resolve().parents[1]
+        k = (root / "deploy/k8s/base/kustomization.yaml").read_text()
+        cron = (root / "deploy/k8s/base/cronjob.yaml").read_text()
+        monitor = (root / "deploy/k8s/base/monitor-deployment.yaml").read_text()
+        self.assertIn("cronjob.yaml", k)
+        self.assertIn("monitor-deployment.yaml", k)
+        self.assertIn("monitor-service.yaml", k)
+        self.assertIn("run-agents", cron)
+        self.assertNotIn("uvicorn", cron.lower())
+        self.assertIn("evaluator_monitor.app:app", monitor)
+        self.assertIn("suspend: false", cron)
+        self.assertIn('timeZone: "America/Sao_Paulo"', cron)
+
+    def test_fastapi_is_confined_to_separate_monitor_package(self):
+        root = Path(__file__).resolve().parents[1]
+        for path in (root / "evaluator").rglob("*.py"):
+            source = path.read_text(encoding="utf-8").lower()
+            self.assertNotIn("import fastapi", source)
+            self.assertNotIn("import uvicorn", source)
+            self.assertNotIn("from fastapi", source)
+        monitor_source = (root / "evaluator_monitor/app.py").read_text(encoding="utf-8").lower()
+        self.assertIn("from fastapi", monitor_source)
+
+
+if __name__ == "__main__":
+    unittest.main()
+
+
+def test_metadata_session_recovery_when_langfuse_sessions_endpoint_is_empty():
+    from evaluator.collectors.langfuse_credentials import LangfuseCredentials
+    from evaluator.config.settings import settings
+
+    class Collector(LangfuseCollector):
+        async def _fetch_sessions(self, client, start, end, creds):
+            return []
+
+        async def _fetch_traces(self, client, start, end, creds):
+            return [
+                {
+                    "id": "trace-real",
+                    "sessionId": None,
+                    "name": "agent.gateway_message",
+                    "metadata": {"agent_id": "telecom_contas"},
+                    "input": "ola",
+                    "output": "resposta",
+                },
+                {
+                    "id": "judge-audit",
+                    "sessionId": None,
+                    "name": "judge-job-telecom_contas",
+                    "metadata": {"agentId": "telecom_contas"},
+                },
+            ]
+
+        async def _fetch_observations(self, client, start, end, creds):
+            return [
+                {
+                    "id": "obs-1",
+                    "traceId": "trace-real",
+                    "metadata": {
+                        "session_id": "default:telecom_contas:session-123",
+                        "agent_id": "telecom_contas",
+                        "message_id": "msg-1",
+                    },
+                }
+            ]
+
+    class NoopAsyncClient:
+        def __init__(self, *args, **kwargs):
+            pass
+        async def __aenter__(self):
+            return self
+        async def __aexit__(self, exc_type, exc, tb):
+            return False
+
+    collector = Collector("telecom_contas")
+    creds = LangfuseCredentials("https://lf.example", "pk", "sk", "test#telecom_contas")
+    old_enable = settings.enable_langfuse
+    try:
+        settings.enable_langfuse = True
+        with patch("evaluator.collectors.langfuse.resolve_langfuse_credentials", return_value=creds), \
+             patch("evaluator.collectors.langfuse.httpx.AsyncClient", NoopAsyncClient):
+            records, sampled, stats = asyncio.run(collector.collect_sampled(
+                datetime(2026, 8, 14, 0, 0),
+                datetime(2026, 8, 14, 23, 59, 59),
+                agent_aliases={"telecom_contas"},
+                percentage=1.0,
+                seed="telecom_contas:2026-08-14",
+            ))
+
+        assert sampled == ["default:telecom_contas:session-123"]
+        assert len(records) == 1
+        assert records[0].trace_id == "trace-real"
+        assert records[0].session_id == "default:telecom_contas:session-123"
+        assert stats["native_sessions_read"] == 0
+        assert stats["metadata_sessions_recovered"] == 1
+        assert stats["sessions_read"] == 1
+        assert stats["traces_without_session"] == 0
+        assert stats["selected_traces"] == 1
+    finally:
+        settings.enable_langfuse = old_enable
+
+
+def test_trace_id_is_never_used_as_session_id_fallback():
+    from evaluator.collectors.langfuse import _trace_session_id
+    assert _trace_session_id({"id": "trace-only", "metadata": {}}) is None

tests/test_monitor_and_run_id.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+
+def test_run_agents_announces_run_id_and_progress_command():
+    source = Path("evaluator/cli.py").read_text(encoding="utf-8")
+    assert "RUN_ID=" in source
+    assert "python -m evaluator.cli progress" in source
+    assert "progress_callback=announce_run" in source
+
+
+def test_monitor_is_separate_from_batch_cronjob():
+    cron = Path("deploy/k8s/base/cronjob.yaml").read_text(encoding="utf-8")
+    monitor = Path("deploy/k8s/base/monitor-deployment.yaml").read_text(encoding="utf-8")
+    service = Path("deploy/k8s/base/monitor-service.yaml").read_text(encoding="utf-8")
+    assert "run-agents" in cron
+    assert "uvicorn" not in cron
+    assert "evaluator_monitor.app:app" in monitor
+    assert "kind: Service" in service
+    assert "type: ClusterIP" in service
+
+
+def test_monitor_routes_present():
+    source = Path("evaluator_monitor/app.py").read_text(encoding="utf-8")
+    for route in ("/health", "/api/runs", "/api/runs/{run_id}", "/api/runs/{run_id}/results", "/ui"):
+        assert route in source

tests/test_no_product_hardcodes.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import ast
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from evaluator.publishers.gcs_export import _load_bucket_mapping, resolve_gcs_bucket
+
+
+ROOT = Path(__file__).resolve().parents[1]
+OFFICIAL_AGENT_IDS = {
+    "billing",
+    "sales",
+    "collection",
+    "backoffice-anatel",
+    "backoffice_anatel",
+    "telecom_contas",
+    "retail_orders",
+    "financeiro_agent",
+}
+
+
+class NoProductHardcodesTests(unittest.TestCase):
+    def test_agent_ids_are_not_python_string_literals_in_runtime(self):
+        offenders = []
+        for path in (ROOT / "evaluator").rglob("*.py"):
+            tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Constant) and isinstance(node.value, str):
+                    if node.value.lower() in OFFICIAL_AGENT_IDS:
+                        offenders.append(f"{path.relative_to(ROOT)}:{node.lineno}: {node.value}")
+        self.assertEqual(offenders, [])
+
+    def test_storage_buckets_are_loaded_from_configuration(self):
+        with tempfile.TemporaryDirectory() as td:
+            cfg = Path(td) / "storage.yaml"
+            cfg.write_text(
+                "storage:\n  buckets:\n    sandbox: bucket-from-config\n",
+                encoding="utf-8",
+            )
+            with patch("evaluator.publishers.gcs_export.settings.storage_config_path", str(cfg)), \
+                 patch("evaluator.publishers.gcs_export.settings.judge_gcs_bucket", None), \
+                 patch("evaluator.publishers.gcs_export.settings.app_env", "sandbox"):
+                self.assertEqual(_load_bucket_mapping(), {"sandbox": "bucket-from-config"})
+                self.assertEqual(resolve_gcs_bucket(), "bucket-from-config")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_tim_spec.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import asyncio
+import unittest
+from datetime import date, datetime
+from pathlib import Path
+
+from evaluator.analytics.vloop import max_consecutive_intent_streak
+from evaluator.config.agents import load_agents
+from evaluator.core.models import ConversationRecord
+from evaluator.engine import EvaluationEngine
+from evaluator.judges.llm_judge import TIMStyleLLMJudge
+from evaluator.output.legacy_exporter import extract_prompt_length
+from evaluator.time_utils import brt_day_window, to_utc_z
+
+
+class FakeLLM:
+    def __init__(self, replies):
+        self.replies = list(replies)
+        self.calls = 0
+
+    async def complete(self, prompt):
+        self.calls += 1
+        return self.replies.pop(0)
+
+
+class TIMSpecTests(unittest.TestCase):
+    def test_brt_day_converts_to_expected_utc(self):
+        start, end = brt_day_window(date(2026, 8, 12))
+        self.assertEqual(to_utc_z(start), "2026-08-12T03:00:00Z")
+        self.assertTrue(to_utc_z(end).startswith("2026-08-13T02:59:59.999999Z"))
+
+    def test_sampling_is_session_based_and_deterministic(self):
+        records = [
+            ConversationRecord(session_id="s1", input_text="a"),
+            ConversationRecord(session_id="s1", input_text="b"),
+            ConversationRecord(session_id="s2", input_text="c"),
+            ConversationRecord(session_id="s3", input_text="d"),
+        ]
+        engine = object.__new__(EvaluationEngine)
+        first, first_sessions = EvaluationEngine._sample_sessions(engine, records, 0.5, "billing", date(2026, 8, 12))
+        second, second_sessions = EvaluationEngine._sample_sessions(engine, records, 0.5, "billing", date(2026, 8, 12))
+        self.assertEqual(first_sessions, second_sessions)
+        self.assertEqual({r.session_id for r in first}, set(first_sessions))
+        for sid in first_sessions:
+            self.assertEqual(
+                sum(1 for r in first if r.session_id == sid),
+                sum(1 for r in records if r.session_id == sid),
+            )
+
+    def test_loop_is_max_consecutive_intent_streak(self):
+        records = [
+            {"metadata": {"intent": "billing"}},
+            {"metadata": {"intent": "billing"}},
+            {"metadata": {"intent": "billing"}},
+            {"metadata": {"intent": "other"}},
+            {"metadata": {"intent": "other"}},
+        ]
+        self.assertEqual(max_consecutive_intent_streak(records), 3)
+
+    def test_prompt_length_is_raw_input_character_count(self):
+        self.assertEqual(extract_prompt_length({"input_text": "12345", "output_text": "ignored"}), 5)
+
+    def test_official_agents_and_prefixes(self):
+        mapping = {a.agent_id: a.gcs_prefix for a in load_agents()}
+        self.assertEqual(mapping, {
+            "billing": "agnt_ai_contas",
+            "sales": "agnt_ai_ofertas",
+            "collection": "agnt_ai_cobranca",
+            "backoffice-anatel": "agnt_ai_bko",
+        })
+
+    def test_cronjob_is_not_suspended(self):
+        text = Path("deploy/k8s/base/cronjob.yaml").read_text()
+        self.assertIn('timeZone: "America/Sao_Paulo"', text)
+        self.assertIn("suspend: false", text)
+
+    def test_trace_parse_retry_success_and_failure(self):
+        async def scenario():
+            llm = FakeLLM([
+                "not-json",
+                '{"judgeScore":0.8,"accuracyScore":0.9,"alucinationScore":0.1,"rationale":"ok"}',
+            ])
+            judge = TIMStyleLLMJudge(llm=llm)
+            result = await judge.judge_trace(ConversationRecord(session_id="s", input_text="x", output_text="y"))
+            self.assertEqual(llm.calls, 2)
+            self.assertEqual(result.judgeScore, 0.8)
+
+            llm2 = FakeLLM(["bad", "still-bad"])
+            judge2 = TIMStyleLLMJudge(llm=llm2)
+            result2 = await judge2.judge_trace(ConversationRecord(session_id="s", input_text="x", output_text="y"))
+            self.assertEqual(llm2.calls, 2)
+            self.assertIsNone(result2.judgeScore)
+            self.assertIsNone(result2.accuracyScore)
+            self.assertIsNone(result2.alucinationScore)
+
+        asyncio.run(scenario())
+
+
+if __name__ == "__main__":
+    unittest.main()