first commit

evaluator/publishers/gcs_export.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from pathlib import Path
+import yaml
+
+from evaluator.config.settings import settings
+
+
+def _load_bucket_mapping() -> dict[str, str]:
+    config_path = settings.path(settings.storage_config_path)
+    if not config_path.exists():
+        raise RuntimeError(
+            f"Storage configuration file not found: {config_path}. "
+            "Set STORAGE_CONFIG_PATH to a YAML file containing storage.buckets."
+        )
+    data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
+    storage = data.get("storage", {}) if isinstance(data, dict) else {}
+    buckets = storage.get("buckets", {}) if isinstance(storage, dict) else {}
+    if not isinstance(buckets, dict):
+        raise RuntimeError(f"Invalid storage.buckets mapping in {config_path}")
+    return {str(k).lower(): str(v) for k, v in buckets.items() if v}
+
+
+def resolve_gcs_bucket() -> str:
+    if settings.judge_gcs_bucket:
+        return settings.judge_gcs_bucket
+    env = (settings.app_env or "").lower()
+    bucket = _load_bucket_mapping().get(env)
+    if not bucket:
+        raise RuntimeError(
+            f"No GCS bucket configured for APP_ENV={settings.app_env!r}. "
+            "Configure storage.buckets in STORAGE_CONFIG_PATH or set JUDGE_GCS_BUCKET."
+        )
+    return bucket
+
+
+def upload_evaluation_file(path: Path, prefix: str = "") -> str:
+    if not settings.enable_gcs_upload:
+        raise RuntimeError("GCS upload requested while ENABLE_GCS_UPLOAD=false")
+
+    from google.cloud import storage
+
+    bucket_name = resolve_gcs_bucket()
+    object_name = "/".join(
+        part.strip("/") for part in (prefix, path.name) if part and part.strip("/")
+    )
+
+    client = storage.Client()
+    bucket = client.bucket(bucket_name)
+    blob = bucket.blob(object_name)
+    blob.upload_from_filename(str(path), content_type="application/gzip")
+    return f"gs://{bucket_name}/{object_name}"

evaluator/publishers/langfuse_job_audit.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timezone
+from typing import Any
+
+import httpx
+
+from evaluator.collectors.langfuse_credentials import resolve_langfuse_credentials
+from evaluator.config.settings import settings
+
+
+class LangfuseJobAuditPublisher:
+    """Audit each agent batch as judge-job-{agentId} in that agent's Langfuse project."""
+
+    def new_trace_id(self) -> str:
+        return uuid.uuid4().hex
+
+    async def publish(
+        self,
+        trace_id: str,
+        agent_id: str,
+        status: str,
+        *,
+        period_start: datetime | None = None,
+        period_end: datetime | None = None,
+        details: dict[str, Any] | None = None,
+    ) -> dict[str, Any] | None:
+        if not settings.enable_langfuse:
+            return None
+        creds = resolve_langfuse_credentials(agent_id)
+        event_timestamp = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+        body: dict[str, Any] = {
+            "id": trace_id,
+            "timestamp": event_timestamp,
+            "name": f"judge-job-{agent_id}",
+            "metadata": {"status": status, "agentId": agent_id, **(details or {})},
+        }
+        if period_start or period_end:
+            body["input"] = {
+                "period_start": period_start.isoformat() if period_start else None,
+                "period_end": period_end.isoformat() if period_end else None,
+            }
+        if status in {"completed", "failed"}:
+            body["output"] = details or {"status": status}
+        payload = {
+            "batch": [{"id": str(uuid.uuid4()), "type": "trace-create", "timestamp": event_timestamp, "body": body}]
+        }
+        async with httpx.AsyncClient(base_url=creds.host, timeout=30) as client:
+            resp = await client.post("/api/public/ingestion", json=payload, auth=creds.auth)
+            if resp.status_code >= 400:
+                raise RuntimeError(f"Langfuse audit failed {resp.status_code}: {resp.text}")
+        return {"ok": True, "trace_id": trace_id, "status": status, "credential_source": creds.source}

evaluator/publishers/langfuse_scores.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+import httpx
+from evaluator.config.settings import settings
+from evaluator.core.models import ConversationRecord, TraceJudgeResult
+
+class LangfuseScorePublisher:
+    async def publish_trace_score(self, record: ConversationRecord, result: TraceJudgeResult):
+        if not settings.can_publish_langfuse_scores or not record.trace_id:
+            return None
+        auth = (settings.langfuse_public_key, settings.langfuse_secret_key)
+        payloads = [
+            {'traceId': record.trace_id, 'name': 'offline_judge_score', 'value': result.judgeScore, 'comment': result.rationale},
+            {'traceId': record.trace_id, 'name': 'offline_accuracy_score', 'value': result.accuracyScore, 'comment': result.rationale},
+            {'traceId': record.trace_id, 'name': 'offline_alucination_score', 'value': result.alucinationScore, 'comment': result.rationale},
+        ]
+        payloads = [payload for payload in payloads if payload['value'] is not None]
+        async with httpx.AsyncClient(base_url=settings.langfuse_host, timeout=30) as client:
+            for payload in payloads:
+                resp = await client.post('/api/public/scores', json=payload, auth=auth)
+                if resp.status_code >= 400:
+                    # Don't fail the run because score publishing is supplementary.
+                    return {'ok': False, 'status': resp.status_code, 'body': resp.text}
+        return {'ok': True}