first commit

deploy/k8s/base/configmap.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: evaluator-config
+  namespace: agent-evaluator
+data:
+  ENABLE_LANGFUSE: "true"
+  LANGFUSE_CREDENTIALS_CONFIG_PATH: "/var/secrets/langfuse/langfuse_credentials.yaml"
+  PUBLISH_LANGFUSE_SCORES: "false"
+  LLM_PROVIDER: "oci_sdk"
+  LLM_PROFILE: "judge"
+  LLM_PROFILES_PATH: "configs/llm_profiles/llm_profiles.yaml"
+  LLM_TEMPERATURE: "0"
+  LLM_MAX_TOKENS: "900"
+  LLM_TIMEOUT_SECONDS: "120"
+  AGENTS_CONFIG_PATH: "configs/judge/agents.yaml"
+  TRACE_PROMPT_PATH: "configs/judge/trace_metrics.yaml"
+  SESSION_PROMPT_PATH: "configs/judge/session_metrics.yaml"
+  OUTPUT_DIR: "/app/output"
+  BATCH_SIZE: "50"
+  MAX_ATTEMPTS: "3"
+  ENABLE_GCS_UPLOAD: "true"
+  STORAGE_CONFIG_PATH: "configs/judge/storage.yaml"
+  ADB_TABLE_PREFIX: "AGENTFW"
+  ADB_WALLET_LOCATION: "/opt/oracle/wallet"

deploy/k8s/base/cronjob.yaml

@@ -0,0 +1,94 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: evaluator-batch
+  namespace: agent-evaluator
+  labels:
+    app.kubernetes.io/name: evaluator-batch
+    app.kubernetes.io/component: batch
+    app.kubernetes.io/part-of: agent-framework
+spec:
+  schedule: "0 2 * * *"
+  timeZone: "America/Sao_Paulo"
+  suspend: false
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 5
+  startingDeadlineSeconds: 1800
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      activeDeadlineSeconds: 21600
+      ttlSecondsAfterFinished: 86400
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: evaluator-batch
+            app.kubernetes.io/component: batch
+        spec:
+          restartPolicy: Never
+          serviceAccountName: evaluator
+          automountServiceAccountToken: false
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 10001
+            runAsGroup: 10001
+            fsGroup: 10001
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: evaluator
+              image: agent-framework-evaluator:latest
+              imagePullPolicy: IfNotPresent
+              command: ["python", "-m", "evaluator.cli", "run-agents"]
+              args: ["--source", "langfuse"]
+              envFrom:
+                - configMapRef:
+                    name: evaluator-config
+                - secretRef:
+                    name: evaluator-secrets
+              env:
+                - name: TNS_ADMIN
+                  value: /opt/oracle/wallet
+              resources:
+                requests:
+                  cpu: 500m
+                  memory: 1Gi
+                limits:
+                  cpu: "2"
+                  memory: 2Gi
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop: ["ALL"]
+              volumeMounts:
+                - name: wallet
+                  mountPath: /opt/oracle/wallet
+                  readOnly: true
+                - name: output
+                  mountPath: /app/output
+                - name: tmp
+                  mountPath: /tmp
+                - name: gcp-credentials
+                  mountPath: /var/secrets/google
+                  readOnly: true
+                - name: langfuse-credentials
+                  mountPath: /var/secrets/langfuse
+                  readOnly: true
+          volumes:
+            - name: wallet
+              secret:
+                secretName: evaluator-wallet
+            - name: output
+              emptyDir: {}
+            - name: tmp
+              emptyDir: {}
+            - name: gcp-credentials
+              secret:
+                secretName: evaluator-gcp
+                optional: true
+            - name: langfuse-credentials
+              secret:
+                secretName: evaluator-langfuse
+                optional: false

deploy/k8s/base/init-db-job.yaml

@@ -0,0 +1,63 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: evaluator-init-db
+  namespace: agent-evaluator
+  labels:
+    app.kubernetes.io/name: evaluator-init-db
+    app.kubernetes.io/component: database-migration
+spec:
+  backoffLimit: 3
+  ttlSecondsAfterFinished: 3600
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: evaluator-init-db
+    spec:
+      restartPolicy: OnFailure
+      serviceAccountName: evaluator
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        runAsGroup: 10001
+        fsGroup: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: init-db
+          image: agent-framework-evaluator:latest
+          imagePullPolicy: IfNotPresent
+          command: ["python", "-m", "evaluator.cli", "init-db"]
+          envFrom:
+            - configMapRef:
+                name: evaluator-config
+            - secretRef:
+                name: evaluator-secrets
+          env:
+            - name: TNS_ADMIN
+              value: /opt/oracle/wallet
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: wallet
+              mountPath: /opt/oracle/wallet
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: wallet
+          secret:
+            secretName: evaluator-wallet
+        - name: tmp
+          emptyDir: {}

deploy/k8s/base/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: agent-evaluator
+resources:
+  - namespace.yaml
+  - serviceaccount.yaml
+  - configmap.yaml
+  - cronjob.yaml
+  - monitor-deployment.yaml
+  - monitor-service.yaml
+commonLabels:
+  app.kubernetes.io/managed-by: kustomize

deploy/k8s/base/monitor-deployment.yaml

@@ -0,0 +1,82 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: evaluator-monitor
+  namespace: agent-evaluator
+  labels:
+    app.kubernetes.io/name: evaluator-monitor
+    app.kubernetes.io/component: monitoring
+    app.kubernetes.io/part-of: agent-framework
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: evaluator-monitor
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: evaluator-monitor
+        app.kubernetes.io/component: monitoring
+    spec:
+      serviceAccountName: evaluator
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        runAsGroup: 10001
+        fsGroup: 10001
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: monitor
+          image: agent-framework-evaluator:latest
+          imagePullPolicy: IfNotPresent
+          command: ["python", "-m", "uvicorn"]
+          args: ["evaluator_monitor.app:app", "--host", "0.0.0.0", "--port", "8080"]
+          ports:
+            - name: http
+              containerPort: 8080
+          envFrom:
+            - configMapRef:
+                name: evaluator-config
+            - secretRef:
+                name: evaluator-secrets
+          env:
+            - name: TNS_ADMIN
+              value: /opt/oracle/wallet
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: wallet
+              mountPath: /opt/oracle/wallet
+              readOnly: true
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: wallet
+          secret:
+            secretName: evaluator-wallet
+        - name: tmp
+          emptyDir: {}

deploy/k8s/base/monitor-service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: evaluator-monitor
+  namespace: agent-evaluator
+  labels:
+    app.kubernetes.io/name: evaluator-monitor
+    app.kubernetes.io/component: monitoring
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: evaluator-monitor
+  ports:
+    - name: http
+      port: 8080
+      targetPort: http

deploy/k8s/base/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: agent-evaluator
+  labels:
+    app.kubernetes.io/part-of: agent-framework

deploy/k8s/base/secret-template.yaml

@@ -0,0 +1,35 @@
+# TEMPLATE ONLY. Do not apply or commit real values.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: evaluator-secrets
+  namespace: agent-evaluator
+type: Opaque
+stringData:
+  ADB_USER: "REPLACE_ME"
+  ADB_PASSWORD: "REPLACE_ME"
+  ADB_DSN: "REPLACE_ME"
+  ADB_WALLET_PASSWORD: "REPLACE_ME"
+  OCI_GENAI_BASE_URL: "https://private-endpoint.inference.generativeai.region.oci.oraclecloud.com"
+  OCI_GENAI_MODEL: "ocid1.generativeaiendpoint.REPLACE_ME"
+  OCI_GENAI_ENDPOINT_ID: "ocid1.generativeaiendpoint.REPLACE_ME"
+  OCI_COMPARTMENT_ID: "ocid1.compartment.REPLACE_ME"
+  OCI_AUTH_MODE: "oke_workload_identity"
+  GOOGLE_APPLICATION_CREDENTIALS: "/var/secrets/google/GCP_ACCESS_KEY.json"
+---
+# Langfuse credentials are one data-driven YAML, not one env namespace per agent.
+# Add as many entries as required; no Python or manifest change is needed.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: evaluator-langfuse
+  namespace: agent-evaluator
+type: Opaque
+stringData:
+  langfuse_credentials.yaml: |
+    langfuse:
+      agents:
+        - agentId: REPLACE_WITH_AGENT_ID
+          host: https://langfuse.example
+          publicKey: pk-REPLACE_ME
+          secretKey: sk-REPLACE_ME

deploy/k8s/base/serviceaccount.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: evaluator
+  namespace: agent-evaluator
+automountServiceAccountToken: false

deploy/k8s/overlays/dev/kustomization.yaml

@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: agent-evaluator-dev
+resources:
+  - ../../base
+images:
+  - name: agent-framework-evaluator
+    newName: REPLACE_REGISTRY/agent-framework-evaluator
+    newTag: dev
+patches:
+  - target:
+      kind: Namespace
+      name: agent-evaluator
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: agent-evaluator-dev
+  - target:
+      kind: CronJob
+      name: evaluator-batch
+    patch: |-
+      - op: replace
+        path: /spec/suspend
+        value: false

deploy/k8s/overlays/hml/kustomization.yaml

@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: agent-evaluator-hml
+resources:
+  - ../../base
+images:
+  - name: agent-framework-evaluator
+    newName: REPLACE_REGISTRY/agent-framework-evaluator
+    newTag: hml
+patches:
+  - target:
+      kind: Namespace
+      name: agent-evaluator
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: agent-evaluator-hml
+  - target:
+      kind: CronJob
+      name: evaluator-batch
+    patch: |-
+      - op: replace
+        path: /spec/suspend
+        value: false

deploy/k8s/overlays/prod/kustomization.yaml

@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: agent-evaluator-prod
+resources:
+  - ../../base
+images:
+  - name: agent-framework-evaluator
+    newName: REPLACE_REGISTRY/agent-framework-evaluator
+    newTag: stable
+patches:
+  - target:
+      kind: Namespace
+      name: agent-evaluator
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: agent-evaluator-prod
+  - target:
+      kind: CronJob
+      name: evaluator-batch
+    patch: |-
+      - op: replace
+        path: /spec/suspend
+        value: false