mirror of
https://github.com/hoshikawa2/agent_platform_oci.git
synced 2026-09-07 18:23:46 +00:00
Testes de Carga: /deploy/oke/load-test
This commit is contained in:
11
deploy/oke/load-test/scripts/00_prepare.sh
Normal file
11
deploy/oke/load-test/scripts/00_prepare.sh
Normal file
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
"$DIR/prepare_env.sh"
|
||||
echo
|
||||
cat <<'TXT'
|
||||
Next:
|
||||
1) edit deploy/oke/load-test/.env.runtime
|
||||
2) copy the real Autonomous wallet into templates/agent_template_backend/wallet/
|
||||
3) run configure_kubeconfig.sh
|
||||
TXT
|
||||
9
deploy/oke/load-test/scripts/01_deploy_all.sh
Normal file
9
deploy/oke/load-test/scripts/01_deploy_all.sh
Normal file
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
"$DIR/deploy_langfuse.sh"
|
||||
"$DIR/build_push_backend.sh"
|
||||
"$DIR/deploy_backend.sh"
|
||||
"$DIR/configure_existing_lb.sh"
|
||||
"$DIR/validate_dependencies.sh"
|
||||
"$DIR/smoke_test.sh" external
|
||||
Binary file not shown.
10
deploy/oke/load-test/scripts/build_push_backend.sh
Normal file
10
deploy/oke/load-test/scripts/build_push_backend.sh
Normal file
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
ENV="$ROOT/deploy/oke/load-test/.env.runtime"
|
||||
set -a; source "$ENV"; set +a
|
||||
IMAGE="${OCI_REGION_KEY}.ocir.io/${OCI_TENANCY_NAMESPACE}/${OCIR_REPOSITORY_PREFIX}/agent-template-backend:${IMAGE_TAG}"
|
||||
echo "Building $IMAGE"
|
||||
docker build -f "$ROOT/deploy/oke/dockerfiles/Dockerfile.agent-template-backend" -t "$IMAGE" "$ROOT"
|
||||
docker push "$IMAGE"
|
||||
echo "$IMAGE" > "$ROOT/deploy/oke/load-test/.backend-image"
|
||||
12
deploy/oke/load-test/scripts/check_load_balancing.sh
Normal file
12
deploy/oke/load-test/scripts/check_load_balancing.sh
Normal file
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
set -a; source "$ROOT/deploy/oke/load-test/.env.runtime"; set +a
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
IP="$(kubectl -n "$NS" get svc agent-template-backend-lb -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
|
||||
[[ -n "$IP" ]] || { echo "LoadBalancer IP not assigned"; exit 3; }
|
||||
URL="http://${IP}:8000/health"
|
||||
echo "Sampling 30 requests from $URL"
|
||||
for i in $(seq 1 30); do
|
||||
curl -fsS -D - -o /dev/null "$URL" 2>/dev/null | awk -F': ' 'tolower($1)=="x-agent-pod" {gsub("\r", "", $2); print $2}'
|
||||
done | sort | uniq -c | sort -nr
|
||||
19
deploy/oke/load-test/scripts/collect_results.sh
Normal file
19
deploy/oke/load-test/scripts/collect_results.sh
Normal file
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
set -a; source "$LT/.env.runtime"; set +a
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
OUT="$LT/results/$STAMP"
|
||||
mkdir -p "$OUT"
|
||||
kubectl -n "$NS" get pods -o wide > "$OUT/pods.txt"
|
||||
kubectl -n "$NS" get hpa -o yaml > "$OUT/hpa.yaml"
|
||||
kubectl -n "$NS" get deploy agent-template-backend -o yaml > "$OUT/deployment.yaml"
|
||||
kubectl -n "$NS" get svc -o wide > "$OUT/services.txt"
|
||||
kubectl -n "$NS" top pods > "$OUT/top-pods.txt" 2>&1 || true
|
||||
kubectl top nodes > "$OUT/top-nodes.txt" 2>&1 || true
|
||||
kubectl -n "$NS" logs job/agent-load-generator > "$OUT/k6.log" 2>&1 || true
|
||||
kubectl -n "$NS" logs -l app=agent-template-backend --prefix --tail=5000 > "$OUT/backend.log" 2>&1 || true
|
||||
kubectl -n "$NS" get events --sort-by='.lastTimestamp' > "$OUT/events.txt" 2>&1 || true
|
||||
echo "$OUT"
|
||||
122
deploy/oke/load-test/scripts/configure_existing_lb.sh
Normal file
122
deploy/oke/load-test/scripts/configure_existing_lb.sh
Normal file
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
ENV="$LT/.env.runtime"
|
||||
[[ -f "$ENV" ]] && source "$ENV"
|
||||
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
SERVICE_NAME="${BACKEND_SERVICE_NAME:-agent-template-backend-lb}"
|
||||
LB_OCID="${EXISTING_LB_OCID:?EXISTING_LB_OCID must be set}"
|
||||
BACKEND_SET="${BACKEND_SET_NAME:-agent-framework-loadtest}"
|
||||
LISTENER_NAME="${BACKEND_LISTENER_NAME:-agent-framework-loadtest}"
|
||||
LISTENER_PORT="${BACKEND_LISTENER_PORT:-8000}"
|
||||
LISTENER_PROTOCOL="${BACKEND_LISTENER_PROTOCOL:-HTTP}"
|
||||
BACKEND_POLICY="${BACKEND_POLICY:-ROUND_ROBIN}"
|
||||
HEALTH_PATH="${BACKEND_HEALTH_PATH:-/health}"
|
||||
OCI_PROFILE="${OCI_CLI_PROFILE:-}"
|
||||
|
||||
log(){ printf '[existing-lb] %s\n' "$*"; }
|
||||
die(){ printf '[existing-lb] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
OCI_ARGS=()
|
||||
[[ -n "$OCI_PROFILE" ]] && OCI_ARGS+=(--profile "$OCI_PROFILE")
|
||||
oci_lb(){ oci lb "$@" "${OCI_ARGS[@]}"; }
|
||||
|
||||
command -v oci >/dev/null || die "oci CLI not found"
|
||||
command -v kubectl >/dev/null || die "kubectl not found"
|
||||
|
||||
log "Validating Load Balancer..."
|
||||
oci_lb load-balancer get --load-balancer-id "$LB_OCID" >/dev/null
|
||||
|
||||
NODE_PORT="$(kubectl -n "$NS" get svc "$SERVICE_NAME" -o jsonpath='{.spec.ports[0].nodePort}')"
|
||||
[[ -n "$NODE_PORT" && "$NODE_PORT" != "0" ]] || die "Service/$SERVICE_NAME has no NodePort"
|
||||
log "NodePort=$NODE_PORT"
|
||||
|
||||
if oci_lb backend-set get --load-balancer-id "$LB_OCID" --backend-set-name "$BACKEND_SET" >/dev/null 2>&1; then
|
||||
log "Backend set '$BACKEND_SET' already exists"
|
||||
else
|
||||
log "Creating backend set '$BACKEND_SET'..."
|
||||
oci_lb backend-set create \
|
||||
--load-balancer-id "$LB_OCID" \
|
||||
--name "$BACKEND_SET" \
|
||||
--policy "$BACKEND_POLICY" \
|
||||
--health-checker-protocol HTTP \
|
||||
--health-checker-port "$NODE_PORT" \
|
||||
--health-checker-url-path "$HEALTH_PATH" \
|
||||
--health-checker-return-code 200 \
|
||||
--health-checker-retries 3 \
|
||||
--health-checker-timeout-in-ms 3000 \
|
||||
--health-checker-interval-in-ms 10000 \
|
||||
--wait-for-state SUCCEEDED \
|
||||
--max-wait-seconds 1200 >/dev/null
|
||||
fi
|
||||
|
||||
for i in $(seq 1 20); do
|
||||
oci_lb backend-set get --load-balancer-id "$LB_OCID" --backend-set-name "$BACKEND_SET" >/dev/null 2>&1 && break
|
||||
[[ "$i" == "20" ]] && die "backend set still unavailable"
|
||||
sleep 3
|
||||
done
|
||||
|
||||
mapfile -t NODE_IPS < <(
|
||||
kubectl get nodes -o jsonpath='{range .items[*]}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' |
|
||||
sed '/^$/d' | sort -u
|
||||
)
|
||||
|
||||
EXISTING="$(oci_lb backend list \
|
||||
--load-balancer-id "$LB_OCID" \
|
||||
--backend-set-name "$BACKEND_SET" \
|
||||
--query 'data[].name' --raw-output 2>/dev/null || true)"
|
||||
|
||||
for ip in "${NODE_IPS[@]}"; do
|
||||
name="${ip}:${NODE_PORT}"
|
||||
if grep -Fxq "$name" <<<"$EXISTING"; then
|
||||
log "Backend $name already exists"
|
||||
else
|
||||
log "Creating backend $name..."
|
||||
oci_lb backend create \
|
||||
--load-balancer-id "$LB_OCID" \
|
||||
--backend-set-name "$BACKEND_SET" \
|
||||
--ip-address "$ip" \
|
||||
--port "$NODE_PORT" \
|
||||
--wait-for-state SUCCEEDED \
|
||||
--max-wait-seconds 1200 >/dev/null
|
||||
fi
|
||||
done
|
||||
|
||||
listener_found="$(
|
||||
oci_lb load-balancer get \
|
||||
--load-balancer-id "$LB_OCID" \
|
||||
--query "data.listeners.\"${LISTENER_NAME}\".name" \
|
||||
--raw-output 2>/dev/null || true
|
||||
)"
|
||||
|
||||
if [[ "$listener_found" == "$LISTENER_NAME" ]]; then
|
||||
log "Listener '$LISTENER_NAME' already exists"
|
||||
else
|
||||
log "Creating listener '$LISTENER_NAME'..."
|
||||
oci_lb listener create \
|
||||
--load-balancer-id "$LB_OCID" \
|
||||
--name "$LISTENER_NAME" \
|
||||
--default-backend-set-name "$BACKEND_SET" \
|
||||
--port "$LISTENER_PORT" \
|
||||
--protocol "$LISTENER_PROTOCOL" \
|
||||
--wait-for-state SUCCEEDED \
|
||||
--max-wait-seconds 1200 >/dev/null
|
||||
fi
|
||||
|
||||
log "Backends:"
|
||||
oci_lb backend list \
|
||||
--load-balancer-id "$LB_OCID" \
|
||||
--backend-set-name "$BACKEND_SET" \
|
||||
--query 'data[].{name:name,ip:"ip-address",port:port}' \
|
||||
--output table
|
||||
|
||||
log "Backend set health:"
|
||||
oci_lb backend-set-health get \
|
||||
--load-balancer-id "$LB_OCID" \
|
||||
--backend-set-name "$BACKEND_SET" \
|
||||
--output table || true
|
||||
|
||||
log "Done"
|
||||
15
deploy/oke/load-test/scripts/configure_kubeconfig.sh
Normal file
15
deploy/oke/load-test/scripts/configure_kubeconfig.sh
Normal file
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
ENV="$ROOT/deploy/oke/load-test/.env.runtime"
|
||||
[[ -f "$ENV" ]] || { echo "Run prepare_env.sh first"; exit 1; }
|
||||
set -a; source "$ENV"; set +a
|
||||
oci ce cluster create-kubeconfig \
|
||||
--cluster-id "$OKE_CLUSTER_OCID" \
|
||||
--file "$HOME/.kube/config" \
|
||||
--region "$OCI_REGION" \
|
||||
--token-version 2.0.0 \
|
||||
--kube-endpoint PUBLIC_ENDPOINT \
|
||||
--profile "${OCI_CLI_PROFILE:-DEFAULT}"
|
||||
kubectl cluster-info
|
||||
kubectl get nodes -o wide
|
||||
10
deploy/oke/load-test/scripts/create_runtime_secret.sh
Normal file
10
deploy/oke/load-test/scripts/create_runtime_secret.sh
Normal file
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
ENV="$ROOT/deploy/oke/load-test/.env.runtime"
|
||||
[[ -f "$ENV" ]] || { echo "Run prepare_env.sh first"; exit 1; }
|
||||
set -a; source "$ENV"; set +a
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f -
|
||||
kubectl -n "$NS" create secret generic agent-backend-runtime --from-env-file="$ENV" --dry-run=client -o yaml | kubectl apply -f -
|
||||
echo "Runtime env loaded into secret agent-backend-runtime."
|
||||
18
deploy/oke/load-test/scripts/create_wallet_secret.sh
Normal file
18
deploy/oke/load-test/scripts/create_wallet_secret.sh
Normal file
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
ENV="$ROOT/deploy/oke/load-test/.env.runtime"
|
||||
WALLET="$ROOT/templates/agent_template_backend/wallet"
|
||||
set -a; source "$ENV"; set +a
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
mapfile -t wallet_files < <(find "$WALLET" -maxdepth 1 -type f ! -name 'README_PLACE_WALLET_HERE.txt' ! -name '.gitkeep')
|
||||
if (( ${#wallet_files[@]} == 0 )); then
|
||||
echo "No real wallet files found in $WALLET"
|
||||
echo "Copy the Autonomous wallet files there, then rerun."
|
||||
exit 2
|
||||
fi
|
||||
kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f -
|
||||
args=()
|
||||
for f in "${wallet_files[@]}"; do args+=("--from-file=$(basename "$f")=$f"); done
|
||||
kubectl -n "$NS" create secret generic agent-backend-wallet "${args[@]}" --dry-run=client -o yaml | kubectl apply -f -
|
||||
echo "Wallet secret agent-backend-wallet created/updated in namespace $NS"
|
||||
91
deploy/oke/load-test/scripts/deploy_backend.sh
Normal file
91
deploy/oke/load-test/scripts/deploy_backend.sh
Normal file
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
ENV="$LT/.env.runtime"
|
||||
|
||||
[[ -f "$ENV" ]] || { echo "ERROR: runtime env not found: $ENV" >&2; exit 1; }
|
||||
|
||||
# Keep variables local to this shell; do not export everything to kubectl/OCI CLI.
|
||||
source "$ENV"
|
||||
|
||||
log() { printf '[deploy-backend] %s\n' "$*"; }
|
||||
die() { printf '[deploy-backend] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
command -v kubectl >/dev/null 2>&1 || die "kubectl not found in PATH"
|
||||
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
OCIR_SECRET_NAME="${OCIR_PULL_SECRET_NAME:-ocir-secret}"
|
||||
|
||||
IMAGE="$(cat "$LT/.backend-image" 2>/dev/null || true)"
|
||||
[[ -n "$IMAGE" ]] || IMAGE="${OCI_REGION_KEY}.ocir.io/${OCI_TENANCY_NAMESPACE}/${OCIR_REPOSITORY_PREFIX}/agent-template-backend:${IMAGE_TAG}"
|
||||
|
||||
log "Namespace: $NS"
|
||||
log "Backend image: $IMAGE"
|
||||
|
||||
log "Checking Kubernetes API authentication..."
|
||||
kubectl get --raw='/readyz' >/dev/null 2>&1 || die "Kubernetes API authentication failed"
|
||||
kubectl get --raw='/openapi/v2' >/dev/null 2>&1 || die "Kubernetes OpenAPI access failed"
|
||||
|
||||
log "Ensuring namespace exists..."
|
||||
kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
|
||||
: "${OCI_REGION_KEY:?OCI_REGION_KEY must be set in $ENV}"
|
||||
: "${OCIR_USERNAME:?OCIR_USERNAME must be set in $ENV}"
|
||||
: "${OCIR_AUTH_TOKEN:?OCIR_AUTH_TOKEN must be set in $ENV}"
|
||||
|
||||
OCIR_EMAIL="${OCIR_EMAIL:-unused@example.invalid}"
|
||||
OCIR_SERVER="${OCI_REGION_KEY}.ocir.io"
|
||||
|
||||
log "Creating/updating OCIR pull secret '$OCIR_SECRET_NAME'..."
|
||||
kubectl create secret docker-registry "$OCIR_SECRET_NAME" -n "$NS" --docker-server="$OCIR_SERVER" --docker-username="$OCIR_USERNAME" --docker-password="$OCIR_AUTH_TOKEN" --docker-email="$OCIR_EMAIL" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
|
||||
log "Associating OCIR pull secret with ServiceAccount/default..."
|
||||
kubectl patch serviceaccount default -n "$NS" --type=merge -p "{\"imagePullSecrets\":[{\"name\":\"${OCIR_SECRET_NAME}\"}]}" >/dev/null
|
||||
|
||||
pull_secret="$(kubectl get serviceaccount default -n "$NS" -o jsonpath='{.imagePullSecrets[*].name}')"
|
||||
[[ " $pull_secret " == *" $OCIR_SECRET_NAME "* ]] || die "ServiceAccount/default does not reference $OCIR_SECRET_NAME"
|
||||
|
||||
log "Creating/updating backend runtime secret..."
|
||||
"$LT/scripts/create_runtime_secret.sh"
|
||||
|
||||
if [[ "${ENABLE_LANGFUSE:-true}" == "true" ]]; then
|
||||
log "Synchronizing Langfuse client credentials..."
|
||||
"$LT/scripts/sync_langfuse_client_secret.sh"
|
||||
fi
|
||||
|
||||
log "Creating/updating backend wallet secret..."
|
||||
"$LT/scripts/create_wallet_secret.sh"
|
||||
|
||||
RUNTIME_CHECKSUM="$(cat "$LT/.env.runtime" "$LT/.env.langfuse" 2>/dev/null | sha256sum | awk '{print $1}')"
|
||||
NODE_PORT="${BACKEND_NODE_PORT:-32116}"
|
||||
|
||||
log "Applying backend Kubernetes manifest..."
|
||||
sed -e "s#namespace: agent-load-test#namespace: $NS#g" -e "s#name: agent-load-test#name: $NS#g" -e "s#IMAGE_PLACEHOLDER#$IMAGE#g" -e "s#replicas: 4#replicas: ${LOADTEST_MIN_REPLICAS:-4}#" -e "s#minReplicas: 4#minReplicas: ${LOADTEST_MIN_REPLICAS:-4}#" -e "s#maxReplicas: 30#maxReplicas: ${LOADTEST_MAX_REPLICAS:-30}#" -e "s#cpu: \"500m\"#cpu: \"${LOADTEST_CPU_REQUEST:-500m}\"#" -e "s#cpu: \"2000m\"#cpu: \"${LOADTEST_CPU_LIMIT:-2000m}\"#" -e "s#memory: \"768Mi\"#memory: \"${LOADTEST_MEMORY_REQUEST:-768Mi}\"#" -e "s#memory: \"2Gi\"#memory: \"${LOADTEST_MEMORY_LIMIT:-2Gi}\"#" -e "s#RUNTIME_CHECKSUM_PLACEHOLDER#$RUNTIME_CHECKSUM#g" -e "s#NODE_PORT_PLACEHOLDER#$NODE_PORT#g" "$LT/k8s/agent-backend.yaml" | kubectl apply -f -
|
||||
|
||||
log "Waiting for backend rollout..."
|
||||
if ! kubectl -n "$NS" rollout status deployment/agent-template-backend --timeout=10m; then
|
||||
echo "================ BACKEND ROLLOUT FAILED ================" >&2
|
||||
kubectl -n "$NS" get pods -o wide >&2 || true
|
||||
bad_pod="$(kubectl -n "$NS" get pods -l app=agent-template-backend --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)"
|
||||
if [[ -n "$bad_pod" ]]; then
|
||||
kubectl -n "$NS" describe pod "$bad_pod" >&2 || true
|
||||
kubectl -n "$NS" logs "$bad_pod" -c agent-template-backend --tail=200 >&2 || true
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Final backend pods:"
|
||||
kubectl -n "$NS" get pods -o wide
|
||||
|
||||
log "Backend NodePort service (fronted by existing OCI LB):"
|
||||
kubectl -n "$NS" get svc agent-template-backend-lb
|
||||
|
||||
log "Backend HPA:"
|
||||
kubectl -n "$NS" get hpa
|
||||
|
||||
log "Default ServiceAccount imagePullSecrets:"
|
||||
kubectl -n "$NS" get serviceaccount default -o jsonpath='{.imagePullSecrets[*].name}{"\n"}'
|
||||
|
||||
log "Backend deployment completed successfully."
|
||||
193
deploy/oke/load-test/scripts/deploy_langfuse.sh
Normal file
193
deploy/oke/load-test/scripts/deploy_langfuse.sh
Normal file
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env bash
|
||||
# deploy_langfuse_v11.sh
|
||||
# OKE-safe Langfuse deployment with DNS preflight.
|
||||
set -Eeuo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
MANIFEST="$LT/langfuse/langfuse-k8s.yaml"
|
||||
LF_ENV="$LT/.env.langfuse"
|
||||
|
||||
NS="${LANGFUSE_NAMESPACE:-langfuse}"
|
||||
SECRET="langfuse-runtime"
|
||||
DNS_TEST_NS="${DNS_TEST_NAMESPACE:-default}"
|
||||
|
||||
PG_FQDN="postgres.${NS}.svc.cluster.local"
|
||||
REDIS_FQDN="redis.${NS}.svc.cluster.local"
|
||||
CH_FQDN="clickhouse.${NS}.svc.cluster.local"
|
||||
MINIO_FQDN="minio.${NS}.svc.cluster.local"
|
||||
MONGO_FQDN="mongo.${NS}.svc.cluster.local"
|
||||
|
||||
log() { printf '[langfuse-deploy] %s\n' "$*"; }
|
||||
die() { printf '[langfuse-deploy] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
cleanup_probe() {
|
||||
local ns="${1:-$DNS_TEST_NS}"
|
||||
local name="${2:-}"
|
||||
[[ -n "$name" ]] && kubectl -n "$ns" delete pod "$name" --ignore-not-found --wait=false >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
trap 'rc=$?; if (( rc != 0 )); then
|
||||
echo >&2
|
||||
echo "[langfuse-deploy] Failure detected." >&2
|
||||
kubectl get nodes -o wide 2>/dev/null >&2 || true
|
||||
kubectl -n kube-system get pods -l k8s-app=kube-dns -o wide 2>/dev/null >&2 || true
|
||||
kubectl -n "$NS" get pods,svc,pvc 2>/dev/null >&2 || true
|
||||
fi
|
||||
exit $rc' EXIT
|
||||
|
||||
command -v kubectl >/dev/null 2>&1 || die "kubectl not found in PATH"
|
||||
[[ -f "$MANIFEST" ]] || die "manifest not found: $MANIFEST"
|
||||
|
||||
log "Checking Kubernetes API..."
|
||||
kubectl version --request-timeout=15s >/dev/null || die "cannot reach Kubernetes API"
|
||||
|
||||
log "Checking worker nodes..."
|
||||
not_ready="$(kubectl get nodes --no-headers 2>/dev/null | awk '$2 != "Ready" {print $1 ":" $2}')"
|
||||
[[ -z "$not_ready" ]] || { printf '%s\n' "$not_ready" >&2; die "one or more OKE nodes are not Ready"; }
|
||||
|
||||
log "Checking CoreDNS..."
|
||||
kubectl -n kube-system get svc kube-dns >/dev/null 2>&1 || die "kube-dns Service not found"
|
||||
dns_ready="$(kubectl -n kube-system get pods -l k8s-app=kube-dns --no-headers 2>/dev/null | awk '$2 ~ /^[0-9]+\/[0-9]+$/ {split($2,a,"/"); if(a[1]==a[2]) n++} END{print n+0}')"
|
||||
(( dns_ready > 0 )) || die "no Ready CoreDNS pod found"
|
||||
|
||||
DNS_IP="$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}')"
|
||||
[[ -n "$DNS_IP" ]] || die "kube-dns ClusterIP is empty"
|
||||
log "kube-dns ClusterIP: $DNS_IP"
|
||||
|
||||
probe="oke-dns-preflight-$(date +%s)"
|
||||
log "Running DNS preflight BEFORE Langfuse deployment..."
|
||||
kubectl -n "$DNS_TEST_NS" run "$probe" --restart=Never --image=docker.io/library/busybox:1.36 --command -- sh -c "echo '--- /etc/resolv.conf ---';
|
||||
cat /etc/resolv.conf;
|
||||
echo '--- default resolver ---';
|
||||
nslookup kubernetes.default.svc.cluster.local;
|
||||
echo '--- direct kube-dns ---';
|
||||
nslookup kubernetes.default.svc.cluster.local ${DNS_IP}" >/dev/null
|
||||
|
||||
if ! kubectl -n "$DNS_TEST_NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$probe" --timeout=90s >/dev/null 2>&1; then
|
||||
echo >&2
|
||||
echo "================ DNS PREFLIGHT FAILED ================" >&2
|
||||
kubectl -n "$DNS_TEST_NS" logs "$probe" >&2 || true
|
||||
kubectl -n "$DNS_TEST_NS" describe pod "$probe" >&2 || true
|
||||
cleanup_probe "$DNS_TEST_NS" "$probe"
|
||||
die "cluster DNS is unavailable; fix worker/CNI networking before deploying Langfuse"
|
||||
fi
|
||||
kubectl -n "$DNS_TEST_NS" logs "$probe" || true
|
||||
cleanup_probe "$DNS_TEST_NS" "$probe"
|
||||
log "Cluster DNS preflight PASSED."
|
||||
|
||||
log "Generating Langfuse runtime environment..."
|
||||
"$LT/scripts/prepare_env.sh"
|
||||
[[ -s "$LF_ENV" ]] || die "Langfuse env was not generated: $LF_ENV"
|
||||
|
||||
required_keys=(
|
||||
LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY
|
||||
LANGFUSE_INIT_PROJECT_PUBLIC_KEY LANGFUSE_INIT_PROJECT_SECRET_KEY
|
||||
LANGFUSE_INIT_ORG_ID LANGFUSE_INIT_ORG_NAME
|
||||
LANGFUSE_INIT_PROJECT_ID LANGFUSE_INIT_PROJECT_NAME
|
||||
LANGFUSE_INIT_USER_EMAIL LANGFUSE_INIT_USER_NAME
|
||||
LANGFUSE_INIT_USER_PASSWORD LANGFUSE_NEXTAUTH_SECRET
|
||||
LANGFUSE_SALT LANGFUSE_ENCRYPTION_KEY
|
||||
LANGFUSE_POSTGRES_PASSWORD LANGFUSE_REDIS_PASSWORD
|
||||
LANGFUSE_CLICKHOUSE_PASSWORD LANGFUSE_MINIO_PASSWORD
|
||||
)
|
||||
|
||||
for key in "${required_keys[@]}"; do
|
||||
grep -qE "^${key}=.+" "$LF_ENV" || die "required key '$key' missing/empty in $LF_ENV"
|
||||
done
|
||||
|
||||
kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
|
||||
pg_password="$(grep '^LANGFUSE_POSTGRES_PASSWORD=' "$LF_ENV" | cut -d= -f2-)"
|
||||
[[ -n "$pg_password" ]] || die "LANGFUSE_POSTGRES_PASSWORD is empty"
|
||||
db_url="postgresql://postgres:${pg_password}@${PG_FQDN}:5432/postgres"
|
||||
|
||||
tmp_env="$(mktemp)"
|
||||
grep -vE '^(DATABASE_URL|DIRECT_URL)=' "$LF_ENV" > "$tmp_env"
|
||||
printf 'DATABASE_URL=%s\n' "$db_url" >> "$tmp_env"
|
||||
printf 'DIRECT_URL=%s\n' "$db_url" >> "$tmp_env"
|
||||
|
||||
log "Creating/updating Secret/$SECRET..."
|
||||
kubectl -n "$NS" create secret generic "$SECRET" --from-env-file="$tmp_env" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
rm -f "$tmp_env"
|
||||
|
||||
log "Applying Langfuse manifest..."
|
||||
kubectl apply -f "$MANIFEST"
|
||||
|
||||
for dep in mongo postgres redis clickhouse minio langfuse-worker langfuse-web; do
|
||||
kubectl -n "$NS" get deployment "$dep" >/dev/null 2>&1 || die "deployment/$dep is missing"
|
||||
done
|
||||
|
||||
pgdata="$(kubectl -n "$NS" get deployment postgres -o jsonpath='{.spec.template.spec.containers[?(@.name=="postgres")].env[?(@.name=="PGDATA")].value}')"
|
||||
[[ "$pgdata" == "/var/lib/postgresql/data/pgdata" ]] || die "Postgres PGDATA is incorrect: '$pgdata'"
|
||||
|
||||
log "Scaling web/worker to zero until infrastructure is healthy..."
|
||||
kubectl -n "$NS" scale deployment/langfuse-web --replicas=0 >/dev/null
|
||||
kubectl -n "$NS" scale deployment/langfuse-worker --replicas=0 >/dev/null
|
||||
|
||||
for dep in mongo postgres redis clickhouse minio; do
|
||||
log "Waiting for deployment/$dep ..."
|
||||
kubectl -n "$NS" rollout status "deployment/$dep" --timeout=10m
|
||||
done
|
||||
|
||||
log "Checking service endpoints..."
|
||||
for svc in mongo postgres redis clickhouse minio; do
|
||||
ip="$(kubectl -n "$NS" get endpoints "$svc" -o jsonpath='{.subsets[0].addresses[0].ip}' 2>/dev/null || true)"
|
||||
[[ -n "$ip" ]] || die "service/$svc has no ready endpoint"
|
||||
log "$svc endpoint: $ip"
|
||||
done
|
||||
|
||||
probe="langfuse-infra-probe-$(date +%s)"
|
||||
log "Running Langfuse infrastructure DNS/TCP probe..."
|
||||
kubectl -n "$NS" run "$probe" --restart=Never --image=docker.io/library/busybox:1.36 --command -- sh -c "set -e;
|
||||
nslookup ${PG_FQDN};
|
||||
nslookup ${REDIS_FQDN};
|
||||
nslookup ${CH_FQDN};
|
||||
nslookup ${MINIO_FQDN};
|
||||
nslookup ${MONGO_FQDN};
|
||||
nc -zvw5 ${PG_FQDN} 5432;
|
||||
nc -zvw5 ${REDIS_FQDN} 6379;
|
||||
nc -zvw5 ${CH_FQDN} 8123;
|
||||
nc -zvw5 ${MINIO_FQDN} 9000;
|
||||
nc -zvw5 ${MONGO_FQDN} 27017;
|
||||
echo 'ALL LANGFUSE INFRA CHECKS PASSED'" >/dev/null
|
||||
|
||||
if ! kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$probe" --timeout=120s >/dev/null 2>&1; then
|
||||
kubectl -n "$NS" logs "$probe" >&2 || true
|
||||
kubectl -n "$NS" describe pod "$probe" >&2 || true
|
||||
cleanup_probe "$NS" "$probe"
|
||||
die "Langfuse infrastructure probe failed; web/worker remain scaled to zero"
|
||||
fi
|
||||
kubectl -n "$NS" logs "$probe" || true
|
||||
cleanup_probe "$NS" "$probe"
|
||||
|
||||
log "Infrastructure healthy. Applying Langfuse web/worker memory sizing..."
|
||||
kubectl -n "$NS" set resources deployment/langfuse-worker --requests=cpu=500m,memory=2Gi --limits=cpu=2,memory=4Gi >/dev/null
|
||||
kubectl -n "$NS" set resources deployment/langfuse-web --requests=cpu=500m,memory=2Gi --limits=cpu=2,memory=4Gi >/dev/null
|
||||
kubectl -n "$NS" set env deployment/langfuse-worker NODE_OPTIONS=--max-old-space-size=3072 >/dev/null
|
||||
kubectl -n "$NS" set env deployment/langfuse-web NODE_OPTIONS=--max-old-space-size=3072 HOSTNAME=0.0.0.0 >/dev/null
|
||||
|
||||
log "Scaling worker/web to 1..."
|
||||
kubectl -n "$NS" scale deployment/langfuse-worker --replicas=1 >/dev/null
|
||||
kubectl -n "$NS" scale deployment/langfuse-web --replicas=1 >/dev/null
|
||||
|
||||
kubectl -n "$NS" rollout status deployment/langfuse-worker --timeout=15m
|
||||
kubectl -n "$NS" rollout status deployment/langfuse-web --timeout=15m
|
||||
|
||||
log "Verifying Secret/$SECRET before synchronization..."
|
||||
kubectl -n "$NS" get secret "$SECRET" >/dev/null || die "Secret/$SECRET became unavailable in namespace $NS"
|
||||
|
||||
APP_NS="$(sed -n -E 's/^K8S_NAMESPACE=(.*)$/\1/p' "$LT/.env.runtime" | tail -n 1 | tr -d '\r' | sed -e 's/^"//' -e 's/"$//' -e "s/^'//" -e "s/'$//")"
|
||||
APP_NS="${APP_NS:-agent-load-test}"
|
||||
|
||||
log "Synchronizing Langfuse project keys to backend namespace $APP_NS..."
|
||||
LANGFUSE_NAMESPACE="$NS" K8S_NAMESPACE="$APP_NS" \
|
||||
"$LT/scripts/sync_langfuse_client_secret.sh"
|
||||
|
||||
log "Validating Langfuse health and authenticated Public API..."
|
||||
"$LT/scripts/validate_langfuse_integration.sh"
|
||||
|
||||
log "Final state:"
|
||||
kubectl -n "$NS" get pods,svc,pvc -o wide
|
||||
printf '\nUI: kubectl -n %s port-forward --address 127.0.0.1 svc/langfuse-web 3005:3000\n' "$NS"
|
||||
log "Langfuse deployment completed successfully."
|
||||
39
deploy/oke/load-test/scripts/fix_oke_worker_network_v1.sh
Normal file
39
deploy/oke/load-test/scripts/fix_oke_worker_network_v1.sh
Normal file
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# fix_oke_worker_network_v1.sh
|
||||
# Run ON EACH OKE WORKER NODE as root or with sudo.
|
||||
set -Eeuo pipefail
|
||||
|
||||
log() { printf '[oke-worker-network] %s\n' "$*"; }
|
||||
|
||||
if [[ "$(id -u)" -ne 0 ]]; then
|
||||
echo "ERROR: run this script with sudo/root." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Loading br_netfilter..."
|
||||
modprobe br_netfilter
|
||||
|
||||
log "Persisting br_netfilter..."
|
||||
cat >/etc/modules-load.d/br_netfilter.conf <<'EOF'
|
||||
br_netfilter
|
||||
EOF
|
||||
|
||||
log "Applying Kubernetes networking sysctls..."
|
||||
cat >/etc/sysctl.d/99-kubernetes-network.conf <<'EOF'
|
||||
net.bridge.bridge-nf-call-iptables=1
|
||||
net.bridge.bridge-nf-call-ip6tables=1
|
||||
net.ipv4.ip_forward=1
|
||||
EOF
|
||||
|
||||
sysctl --system >/dev/null
|
||||
|
||||
log "Validating kernel/module settings..."
|
||||
lsmod | grep -q '^br_netfilter' || { echo "ERROR: br_netfilter is not loaded." >&2; exit 1; }
|
||||
[[ "$(sysctl -n net.bridge.bridge-nf-call-iptables)" == "1" ]] || { echo "ERROR: bridge-nf-call-iptables != 1" >&2; exit 1; }
|
||||
[[ "$(sysctl -n net.ipv4.ip_forward)" == "1" ]] || { echo "ERROR: ip_forward != 1" >&2; exit 1; }
|
||||
|
||||
log "Restarting CRI-O and kubelet..."
|
||||
systemctl restart crio
|
||||
systemctl restart kubelet
|
||||
|
||||
log "Worker network correction completed."
|
||||
151
deploy/oke/load-test/scripts/prepare_env.py
Normal file
151
deploy/oke/load-test/scripts/prepare_env.py
Normal file
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shlex
|
||||
|
||||
|
||||
def parse_env(path: Path) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
if not path.exists():
|
||||
return out
|
||||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
# Support the common `export KEY=value` form too.
|
||||
if line.startswith("export "):
|
||||
line = line[7:].lstrip()
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
if not k:
|
||||
continue
|
||||
out[k] = v.strip()
|
||||
return out
|
||||
|
||||
|
||||
def write_env(path: Path, values: dict[str, str], *, shell_safe: bool = False):
|
||||
lines = ["# Generated by prepare_env.py. Do not commit."]
|
||||
for k in sorted(values):
|
||||
value = values[k]
|
||||
if shell_safe:
|
||||
value = shlex.quote(value)
|
||||
lines.append(f"{k}={value}")
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def merge_missing(target: dict[str, str], defaults: dict[str, str]) -> None:
|
||||
"""Add only keys that are missing from target; never replace backend values."""
|
||||
for k, v in defaults.items():
|
||||
target.setdefault(k, v)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base", required=True,
|
||||
help="Main application env. Existing values are preserved.")
|
||||
p.add_argument("--overlay", required=True,
|
||||
help="Load-test defaults. Only fills keys absent from --base.")
|
||||
p.add_argument("--override", required=False,
|
||||
help="Optional explicit load-test overrides. These replace base/default values.")
|
||||
p.add_argument("--output", required=True)
|
||||
p.add_argument("--langfuse-env", required=True)
|
||||
args = p.parse_args()
|
||||
|
||||
base_path = Path(args.base)
|
||||
defaults_path = Path(args.overlay)
|
||||
override_path = Path(args.override) if args.override else None
|
||||
|
||||
base = parse_env(base_path)
|
||||
defaults = parse_env(defaults_path)
|
||||
explicit = parse_env(override_path) if override_path and override_path.exists() else {}
|
||||
|
||||
# Backend .env is authoritative. The example file provides only missing load-test keys.
|
||||
merged = dict(base)
|
||||
merge_missing(merged, defaults)
|
||||
|
||||
# .env.loadtest, when present, is the deliberate operator override layer.
|
||||
merged.update(explicit)
|
||||
|
||||
# Container/OKE invariants. These are location/routing details that cannot use
|
||||
# workstation-local values from templates/agent_template_backend/.env.
|
||||
merged["APP_ENV"] = "oke-load-test"
|
||||
merged["ADB_WALLET_LOCATION"] = "/app/wallet"
|
||||
merged["ENABLE_LANGFUSE"] = "true"
|
||||
merged["LANGFUSE_HOST"] = "http://langfuse-web.langfuse.svc.cluster.local:3000"
|
||||
|
||||
# OKE authentication should normally use workload identity. Allow an explicit
|
||||
# .env.loadtest override if the operator intentionally selected another mode.
|
||||
if "OCI_AUTH_MODE" not in explicit:
|
||||
merged["OCI_AUTH_MODE"] = "oke_workload_identity"
|
||||
|
||||
# Correct only the historical typo in Mongo-compatible ports; preserve host,
|
||||
# credentials, query params and database values from the backend .env.
|
||||
for key in ("MONGODB_URI", "PUBSUB_SEQUENCE_MONGODB_URI"):
|
||||
if key in merged:
|
||||
merged[key] = merged[key].replace(":37017", ":27017")
|
||||
|
||||
# Langfuse bootstrap secrets are stable across reruns.
|
||||
lf_path = Path(args.langfuse_env)
|
||||
lf = parse_env(lf_path)
|
||||
public_key = lf.get("LANGFUSE_PUBLIC_KEY") or "pk-lf-" + secrets.token_hex(16)
|
||||
secret_key = lf.get("LANGFUSE_SECRET_KEY") or "sk-lf-" + secrets.token_hex(32)
|
||||
admin_password = lf.get("LANGFUSE_INIT_USER_PASSWORD") or secrets.token_urlsafe(24)
|
||||
|
||||
lf.update({
|
||||
"LANGFUSE_INIT_ORG_ID": lf.get("LANGFUSE_INIT_ORG_ID", "agent-framework-org"),
|
||||
"LANGFUSE_INIT_ORG_NAME": lf.get("LANGFUSE_INIT_ORG_NAME", "Agent Framework OCI"),
|
||||
"LANGFUSE_INIT_PROJECT_ID": lf.get("LANGFUSE_INIT_PROJECT_ID", "agent-framework-loadtest"),
|
||||
"LANGFUSE_INIT_PROJECT_NAME": lf.get("LANGFUSE_INIT_PROJECT_NAME", "Agent Framework Load Test"),
|
||||
"LANGFUSE_PUBLIC_KEY": public_key,
|
||||
"LANGFUSE_SECRET_KEY": secret_key,
|
||||
"LANGFUSE_INIT_USER_EMAIL": lf.get("LANGFUSE_INIT_USER_EMAIL", "admin@loadtest.local"),
|
||||
"LANGFUSE_INIT_USER_NAME": lf.get("LANGFUSE_INIT_USER_NAME", "Load Test Admin"),
|
||||
"LANGFUSE_INIT_USER_PASSWORD": admin_password,
|
||||
"LANGFUSE_NEXTAUTH_SECRET": lf.get("LANGFUSE_NEXTAUTH_SECRET") or secrets.token_hex(32),
|
||||
"LANGFUSE_SALT": lf.get("LANGFUSE_SALT") or secrets.token_hex(16),
|
||||
"LANGFUSE_ENCRYPTION_KEY": lf.get("LANGFUSE_ENCRYPTION_KEY") or secrets.token_hex(32),
|
||||
"LANGFUSE_POSTGRES_PASSWORD": lf.get("LANGFUSE_POSTGRES_PASSWORD") or secrets.token_urlsafe(24),
|
||||
"LANGFUSE_REDIS_PASSWORD": lf.get("LANGFUSE_REDIS_PASSWORD") or secrets.token_urlsafe(24),
|
||||
"LANGFUSE_CLICKHOUSE_PASSWORD": lf.get("LANGFUSE_CLICKHOUSE_PASSWORD") or secrets.token_urlsafe(24),
|
||||
"LANGFUSE_MINIO_PASSWORD": lf.get("LANGFUSE_MINIO_PASSWORD") or secrets.token_urlsafe(24),
|
||||
})
|
||||
lf.update({
|
||||
"DATABASE_URL": f"postgresql://postgres:{lf['LANGFUSE_POSTGRES_PASSWORD']}@postgres.langfuse.svc.cluster.local:5432/postgres",
|
||||
"DIRECT_URL": f"postgresql://postgres:{lf['LANGFUSE_POSTGRES_PASSWORD']}@postgres.langfuse.svc.cluster.local:5432/postgres",
|
||||
"POSTGRES_PASSWORD": lf["LANGFUSE_POSTGRES_PASSWORD"],
|
||||
"SALT": lf["LANGFUSE_SALT"],
|
||||
"ENCRYPTION_KEY": lf["LANGFUSE_ENCRYPTION_KEY"],
|
||||
"NEXTAUTH_SECRET": lf["LANGFUSE_NEXTAUTH_SECRET"],
|
||||
"CLICKHOUSE_PASSWORD": lf["LANGFUSE_CLICKHOUSE_PASSWORD"],
|
||||
"REDIS_AUTH": lf["LANGFUSE_REDIS_PASSWORD"],
|
||||
"LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY": lf["LANGFUSE_MINIO_PASSWORD"],
|
||||
"LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY": lf["LANGFUSE_MINIO_PASSWORD"],
|
||||
"LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY": lf["LANGFUSE_MINIO_PASSWORD"],
|
||||
"LANGFUSE_INIT_PROJECT_PUBLIC_KEY": public_key,
|
||||
"LANGFUSE_INIT_PROJECT_SECRET_KEY": secret_key,
|
||||
})
|
||||
write_env(lf_path, lf, shell_safe=False)
|
||||
|
||||
merged["LANGFUSE_PUBLIC_KEY"] = public_key
|
||||
merged["LANGFUSE_SECRET_KEY"] = secret_key
|
||||
write_env(Path(args.output), merged, shell_safe=True)
|
||||
|
||||
preserved = sum(1 for k, v in base.items() if merged.get(k) == v)
|
||||
changed = sorted(k for k, v in base.items() if merged.get(k) != v)
|
||||
print(f"Base env: {base_path} ({len(base)} variables)")
|
||||
print(f"Defaults: {defaults_path} ({len(defaults)} variables)")
|
||||
if override_path:
|
||||
print(f"Explicit override: {override_path} ({len(explicit)} variables)")
|
||||
print(f"Runtime env written: {args.output} ({len(merged)} variables)")
|
||||
print(f"Backend values preserved unchanged: {preserved}/{len(base)}")
|
||||
if changed:
|
||||
print("Backend values intentionally changed for OKE/container compatibility:")
|
||||
for k in changed:
|
||||
print(f" - {k}")
|
||||
print(f"Langfuse env written: {args.langfuse_env}")
|
||||
print("LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY synchronized automatically.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
31
deploy/oke/load-test/scripts/prepare_env.sh
Normal file
31
deploy/oke/load-test/scripts/prepare_env.sh
Normal file
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
BASE="$ROOT/templates/agent_template_backend/.env"
|
||||
DEFAULTS="$LT/.env.loadtest.example"
|
||||
OVERRIDE="$LT/.env.loadtest"
|
||||
|
||||
[[ -f "$BASE" ]] || { echo "ERROR: backend env not found: $BASE" >&2; exit 1; }
|
||||
[[ -f "$DEFAULTS" ]] || { echo "ERROR: load-test defaults not found: $DEFAULTS" >&2; exit 1; }
|
||||
|
||||
args=(
|
||||
--base "$BASE"
|
||||
--overlay "$DEFAULTS"
|
||||
--output "$LT/.env.runtime"
|
||||
--langfuse-env "$LT/.env.langfuse"
|
||||
)
|
||||
|
||||
if [[ -f "$OVERRIDE" ]]; then
|
||||
args+=(--override "$OVERRIDE")
|
||||
echo "Using explicit load-test overrides: $OVERRIDE"
|
||||
else
|
||||
echo "No $OVERRIDE found; backend .env values will be preserved and example defaults only fill missing keys."
|
||||
fi
|
||||
|
||||
python3 "$LT/scripts/prepare_env.py" "${args[@]}"
|
||||
|
||||
echo
|
||||
echo "Runtime environment prepared: $LT/.env.runtime"
|
||||
echo "Source of application configuration: $BASE"
|
||||
echo "Optional explicit overrides: $OVERRIDE"
|
||||
37
deploy/oke/load-test/scripts/run_architecture_suite.sh
Normal file
37
deploy/oke/load-test/scripts/run_architecture_suite.sh
Normal file
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
ENV="$LT/.env.runtime"
|
||||
[[ -f "$ENV" ]] || "$LT/scripts/prepare_env.sh"
|
||||
set -a; source "$ENV"; set +a
|
||||
|
||||
run_case() {
|
||||
local name="$1" rps="$2" dur="$3" scenario="$4"
|
||||
echo "=== $name: ${rps} rps / ${dur} / ${scenario} ==="
|
||||
python3 - "$ENV" "$rps" "$dur" "$scenario" <<'PY'
|
||||
import sys
|
||||
p,rps,dur,scenario=sys.argv[1:]
|
||||
kv={}
|
||||
order=[]
|
||||
for line in open(p):
|
||||
if '=' in line and not line.startswith('#'):
|
||||
k,v=line.rstrip('\n').split('=',1); kv[k]=v; order.append(k)
|
||||
kv['LOADTEST_RPS']=rps; kv['LOADTEST_DURATION']=dur; kv['LOADTEST_SCENARIO']=scenario
|
||||
for k in ('LOADTEST_RPS','LOADTEST_DURATION','LOADTEST_SCENARIO'):
|
||||
if k not in order: order.append(k)
|
||||
with open(p,'w') as f:
|
||||
f.write('# Generated/updated load-test runtime env. Do not commit.\n')
|
||||
for k in order: f.write(f'{k}={kv[k]}\n')
|
||||
PY
|
||||
"$LT/scripts/run_load_test.sh" internal
|
||||
kubectl -n "${K8S_NAMESPACE:-agent-load-test}" logs -f job/agent-load-generator || true
|
||||
"$LT/scripts/collect_results.sh"
|
||||
}
|
||||
|
||||
run_case warmup 5 2m unique_sessions
|
||||
run_case baseline 25 5m unique_sessions
|
||||
run_case scale 100 10m unique_sessions
|
||||
run_case shared-state 50 10m shared_sessions
|
||||
|
||||
echo "Suite completed. Review deploy/oke/load-test/results and Langfuse."
|
||||
111
deploy/oke/load-test/scripts/run_load_test.sh
Normal file
111
deploy/oke/load-test/scripts/run_load_test.sh
Normal file
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
ENV="$LT/.env.runtime"
|
||||
[[ -f "$ENV" ]] || { echo "ERROR: $ENV not found" >&2; exit 1; }
|
||||
source "$ENV"
|
||||
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
TARGET_MODE="${1:-${LOADTEST_TARGET_MODE:-internal}}"
|
||||
|
||||
resolve_external_url() {
|
||||
if [[ -n "${LOADTEST_EXTERNAL_URL:-}" ]]; then
|
||||
printf '%s\n' "${LOADTEST_EXTERNAL_URL%/}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
: "${EXISTING_LB_OCID:?Set EXISTING_LB_OCID or LOADTEST_EXTERNAL_URL in .env.runtime}"
|
||||
|
||||
command -v oci >/dev/null 2>&1 || {
|
||||
echo "ERROR: oci CLI not found and LOADTEST_EXTERNAL_URL is not set" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
local oci_args=()
|
||||
[[ -n "${OCI_CLI_PROFILE:-}" ]] && oci_args+=(--profile "$OCI_CLI_PROFILE")
|
||||
|
||||
local ip
|
||||
ip="$(
|
||||
oci lb load-balancer get \
|
||||
--load-balancer-id "$EXISTING_LB_OCID" \
|
||||
"${oci_args[@]}" \
|
||||
--query 'data."ip-addresses"[0]."ip-address"' \
|
||||
--raw-output
|
||||
)"
|
||||
|
||||
[[ -n "$ip" && "$ip" != "null" ]] || {
|
||||
echo "ERROR: could not resolve IP from existing LB $EXISTING_LB_OCID" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
local proto="${BACKEND_LISTENER_PROTOCOL:-HTTP}"
|
||||
local scheme
|
||||
scheme="$(printf '%s' "$proto" | tr '[:upper:]' '[:lower:]')"
|
||||
[[ "$scheme" == "http" || "$scheme" == "https" ]] || scheme="http"
|
||||
|
||||
local port="${BACKEND_LISTENER_PORT:-8000}"
|
||||
printf '%s://%s:%s\n' "$scheme" "$ip" "$port"
|
||||
}
|
||||
|
||||
case "$TARGET_MODE" in
|
||||
external)
|
||||
LOADTEST_TARGET="$(resolve_external_url)"
|
||||
;;
|
||||
internal)
|
||||
LOADTEST_TARGET="http://agent-template-backend.${NS}.svc.cluster.local:8000"
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 [internal|external]" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Load-test mode: $TARGET_MODE"
|
||||
echo "Load-test target: $LOADTEST_TARGET"
|
||||
|
||||
TMP_ENV="$(mktemp)"
|
||||
trap 'rm -f "$TMP_ENV"' EXIT
|
||||
cp "$ENV" "$TMP_ENV"
|
||||
|
||||
python3 - "$TMP_ENV" "$LOADTEST_TARGET" "$TARGET_MODE" <<'PY'
|
||||
import sys
|
||||
p,target,mode=sys.argv[1:]
|
||||
lines=[]
|
||||
seen_target=False
|
||||
seen_mode=False
|
||||
for line in open(p):
|
||||
if line.startswith("LOADTEST_TARGET="):
|
||||
lines.append(f"LOADTEST_TARGET={target}\n")
|
||||
seen_target=True
|
||||
elif line.startswith("LOADTEST_TARGET_MODE="):
|
||||
lines.append(f"LOADTEST_TARGET_MODE={mode}\n")
|
||||
seen_mode=True
|
||||
else:
|
||||
lines.append(line)
|
||||
if not seen_target:
|
||||
lines.append(f"LOADTEST_TARGET={target}\n")
|
||||
if not seen_mode:
|
||||
lines.append(f"LOADTEST_TARGET_MODE={mode}\n")
|
||||
open(p,"w").writelines(lines)
|
||||
PY
|
||||
|
||||
kubectl -n "$NS" create secret generic agent-backend-runtime \
|
||||
--from-env-file="$TMP_ENV" \
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
kubectl -n "$NS" create configmap k6-load-script \
|
||||
--from-file=loadtest.js="$LT/loadgen/loadtest.js" \
|
||||
--dry-run=client -o yaml | kubectl apply -f -
|
||||
|
||||
kubectl -n "$NS" delete job agent-load-generator \
|
||||
--ignore-not-found --wait=true
|
||||
|
||||
sed \
|
||||
-e "s#namespace: agent-load-test#namespace: $NS#g" \
|
||||
"$LT/k8s/k6-job.yaml" | kubectl apply -f -
|
||||
|
||||
echo "Load test started against $LOADTEST_TARGET"
|
||||
echo "Follow: kubectl -n $NS logs -f job/agent-load-generator"
|
||||
echo "Watch: watch kubectl -n $NS get pods,hpa"
|
||||
27
deploy/oke/load-test/scripts/set_llm_mode.sh
Normal file
27
deploy/oke/load-test/scripts/set_llm_mode.sh
Normal file
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
ENV="$ROOT/deploy/oke/load-test/.env.runtime"
|
||||
MODE="${1:-}"
|
||||
case "$MODE" in
|
||||
mock) VALUE=mock ;;
|
||||
real|oci) VALUE=oci_sdk ;;
|
||||
*) echo "Usage: $0 mock|real"; exit 2 ;;
|
||||
esac
|
||||
python3 - "$ENV" "$VALUE" <<'PY'
|
||||
import sys
|
||||
p,v=sys.argv[1:]
|
||||
lines=open(p).read().splitlines(); out=[]; found=False
|
||||
for line in lines:
|
||||
if line.startswith('LLM_PROVIDER='):
|
||||
out.append('LLM_PROVIDER='+v); found=True
|
||||
else: out.append(line)
|
||||
if not found: out.append('LLM_PROVIDER='+v)
|
||||
open(p,'w').write('\n'.join(out)+'\n')
|
||||
PY
|
||||
set -a; source "$ENV"; set +a
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
kubectl -n "$NS" create secret generic agent-backend-runtime --from-env-file="$ENV" --dry-run=client -o yaml | kubectl apply -f -
|
||||
kubectl -n "$NS" rollout restart deployment/agent-template-backend
|
||||
kubectl -n "$NS" rollout status deployment/agent-template-backend --timeout=10m
|
||||
echo "LLM_PROVIDER=$VALUE"
|
||||
131
deploy/oke/load-test/scripts/smoke_test.sh
Normal file
131
deploy/oke/load-test/scripts/smoke_test.sh
Normal file
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
ENV="$ROOT/deploy/oke/load-test/.env.runtime"
|
||||
[[ -f "$ENV" ]] || { echo "ERROR: $ENV not found" >&2; exit 1; }
|
||||
source "$ENV"
|
||||
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
MODE="${1:-external}"
|
||||
|
||||
resolve_external_url() {
|
||||
if [[ -n "${LOADTEST_EXTERNAL_URL:-}" ]]; then
|
||||
printf '%s\n' "${LOADTEST_EXTERNAL_URL%/}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
: "${EXISTING_LB_OCID:?Set EXISTING_LB_OCID or LOADTEST_EXTERNAL_URL in .env.runtime}"
|
||||
|
||||
command -v oci >/dev/null 2>&1 || {
|
||||
echo "ERROR: oci CLI not found and LOADTEST_EXTERNAL_URL is not set" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
local oci_args=()
|
||||
[[ -n "${OCI_CLI_PROFILE:-}" ]] && oci_args+=(--profile "$OCI_CLI_PROFILE")
|
||||
|
||||
local ip
|
||||
ip="$(
|
||||
oci lb load-balancer get \
|
||||
--load-balancer-id "$EXISTING_LB_OCID" \
|
||||
"${oci_args[@]}" \
|
||||
--query 'data."ip-addresses"[0]."ip-address"' \
|
||||
--raw-output
|
||||
)"
|
||||
|
||||
[[ -n "$ip" && "$ip" != "null" ]] || {
|
||||
echo "ERROR: could not resolve IP from existing LB $EXISTING_LB_OCID" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
local proto="${BACKEND_LISTENER_PROTOCOL:-HTTP}"
|
||||
local scheme
|
||||
scheme="$(printf '%s' "$proto" | tr '[:upper:]' '[:lower:]')"
|
||||
[[ "$scheme" == "http" || "$scheme" == "https" ]] || scheme="http"
|
||||
|
||||
local port="${BACKEND_LISTENER_PORT:-8000}"
|
||||
printf '%s://%s:%s\n' "$scheme" "$ip" "$port"
|
||||
}
|
||||
|
||||
if [[ "$MODE" == "external" ]]; then
|
||||
URL="$(resolve_external_url)"
|
||||
elif [[ "$MODE" == "internal" ]]; then
|
||||
# Execute curl inside the cluster because *.svc.cluster.local is not resolvable
|
||||
# from the developer workstation.
|
||||
URL="http://agent-template-backend.${NS}.svc.cluster.local:8000"
|
||||
else
|
||||
echo "Usage: $0 [external|internal]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
SID="smoke-$(date +%s)"
|
||||
PAYLOAD="{\"channel\":\"web\",\"agent_id\":\"${LOADTEST_AGENT_ID:-telecom_contas}\",\"tenant_id\":\"loadtest\",\"payload\":{\"text\":\"Olá. Responda apenas OK.\",\"session_id\":\"$SID\",\"user_id\":\"smoke\",\"customer_id\":\"smoke\",\"message_id\":\"smoke-1\"}}"
|
||||
|
||||
|
||||
verify_langfuse_trace() {
|
||||
[[ "${ENABLE_LANGFUSE:-true}" == "true" ]] || return 0
|
||||
local checkpod="langfuse-trace-check-$(date +%s)"
|
||||
local needle="$SID"
|
||||
echo "Verifying trace ingestion in Langfuse for session $needle ..."
|
||||
kubectl -n "$NS" delete pod "$checkpod" --ignore-not-found >/dev/null 2>&1 || true
|
||||
kubectl -n "$NS" run "$checkpod" --restart=Never --image=docker.io/curlimages/curl:latest \
|
||||
--overrides="$(cat <<JSON
|
||||
{"spec":{"containers":[{"name":"$checkpod","image":"docker.io/curlimages/curl:latest","env":[
|
||||
{"name":"LANGFUSE_HOST","valueFrom":{"secretKeyRef":{"name":"langfuse-client","key":"LANGFUSE_HOST"}}},
|
||||
{"name":"LANGFUSE_PUBLIC_KEY","valueFrom":{"secretKeyRef":{"name":"langfuse-client","key":"LANGFUSE_PUBLIC_KEY"}}},
|
||||
{"name":"LANGFUSE_SECRET_KEY","valueFrom":{"secretKeyRef":{"name":"langfuse-client","key":"LANGFUSE_SECRET_KEY"}}}
|
||||
],"command":["sh","-c"],"args":["set -eu; i=0; while [ \$i -lt 12 ]; do body=\$(curl -fsS -u \"\$LANGFUSE_PUBLIC_KEY:\$LANGFUSE_SECRET_KEY\" \"\$LANGFUSE_HOST/api/public/traces?limit=100\" || true); echo \"\$body\" | grep -F '$needle' >/dev/null && { echo LANGFUSE_TRACE_OK; exit 0; }; i=\$((i+1)); sleep 5; done; echo LANGFUSE_TRACE_NOT_FOUND >&2; exit 1"]}]}}
|
||||
JSON
|
||||
)" >/dev/null
|
||||
if ! kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$checkpod" --timeout=90s >/dev/null 2>&1; then
|
||||
kubectl -n "$NS" logs "$checkpod" || true
|
||||
kubectl -n "$NS" delete pod "$checkpod" --ignore-not-found >/dev/null 2>&1 || true
|
||||
return 1
|
||||
fi
|
||||
kubectl -n "$NS" logs "$checkpod"
|
||||
kubectl -n "$NS" delete pod "$checkpod" --ignore-not-found >/dev/null
|
||||
}
|
||||
|
||||
echo "Smoke target: $URL ($MODE)"
|
||||
|
||||
if [[ "$MODE" == "external" ]]; then
|
||||
echo "Health:"
|
||||
curl -fsS --connect-timeout 10 --max-time 30 "$URL/health"
|
||||
echo
|
||||
echo "Gateway message:"
|
||||
curl -fsS --connect-timeout 10 --max-time 120 \
|
||||
-X POST "$URL/gateway/message" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H "X-Request-ID: smoke-$SID" \
|
||||
-d "$PAYLOAD"
|
||||
echo
|
||||
else
|
||||
pod="backend-smoke-$(date +%s)"
|
||||
kubectl -n "$NS" run "$pod" \
|
||||
--restart=Never \
|
||||
--image=docker.io/curlimages/curl:latest \
|
||||
--command -- sh -c \
|
||||
"set -e;
|
||||
echo 'Health:';
|
||||
curl -fsS --connect-timeout 10 --max-time 30 '$URL/health';
|
||||
echo;
|
||||
echo 'Gateway message:';
|
||||
curl -fsS --connect-timeout 10 --max-time 120 -X POST '$URL/gateway/message' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H 'X-Request-ID: smoke-$SID' \
|
||||
-d '$PAYLOAD';
|
||||
echo" >/dev/null
|
||||
|
||||
if ! kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$pod" --timeout=180s >/dev/null 2>&1; then
|
||||
kubectl -n "$NS" logs "$pod" || true
|
||||
kubectl -n "$NS" describe pod "$pod" || true
|
||||
kubectl -n "$NS" delete pod "$pod" --ignore-not-found >/dev/null 2>&1 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
kubectl -n "$NS" logs "$pod"
|
||||
kubectl -n "$NS" delete pod "$pod" --ignore-not-found >/dev/null
|
||||
fi
|
||||
|
||||
verify_langfuse_trace
|
||||
11
deploy/oke/load-test/scripts/start_langfuse_compose.sh
Normal file
11
deploy/oke/load-test/scripts/start_langfuse_compose.sh
Normal file
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
[[ -f "$LT/.env.langfuse" ]] || "$LT/scripts/prepare_env.sh"
|
||||
docker compose \
|
||||
--env-file "$LT/.env.langfuse" \
|
||||
-f "$ROOT/libs/agent_framework/Infrastructure_Langfuse/docker-compose.yml" \
|
||||
up -d
|
||||
echo "Local Langfuse: http://localhost:3005"
|
||||
echo "Keys are synchronized in $LT/.env.langfuse and $LT/.env.runtime"
|
||||
91
deploy/oke/load-test/scripts/sync_langfuse_client_secret.sh
Normal file
91
deploy/oke/load-test/scripts/sync_langfuse_client_secret.sh
Normal file
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
RUNTIME_ENV="$LT/.env.runtime"
|
||||
LF_ENV="$LT/.env.langfuse"
|
||||
|
||||
log(){ printf '[langfuse-sync] %s\n' "$*"; }
|
||||
die(){ printf '[langfuse-sync] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
# IMPORTANT: do not `source .env.runtime` here. It contains backend OCI settings
|
||||
# (OCI_AUTH_MODE, OCI_PROFILE, OCI_CLI_PROFILE, etc.) that can change the OCI CLI
|
||||
# exec-plugin environment used by kubectl/kubeconfig.
|
||||
read_env_value() {
|
||||
local file="$1" key="$2" default_value="${3:-}"
|
||||
local value=""
|
||||
if [[ -f "$file" ]]; then
|
||||
value="$(sed -n -E "s/^${key}=(.*)$/\\1/p" "$file" | tail -n 1)"
|
||||
value="${value%$'\r'}"
|
||||
if [[ "$value" == \"*\" && "$value" == *\" ]]; then value="${value:1:${#value}-2}"; fi
|
||||
if [[ "$value" == \'*\' && "$value" == *\' ]]; then value="${value:1:${#value}-2}"; fi
|
||||
fi
|
||||
printf '%s\n' "${value:-$default_value}"
|
||||
}
|
||||
|
||||
LF_NS="${LANGFUSE_NAMESPACE:-$(read_env_value "$RUNTIME_ENV" LANGFUSE_NAMESPACE langfuse)}"
|
||||
APP_NS="${K8S_NAMESPACE:-$(read_env_value "$RUNTIME_ENV" K8S_NAMESPACE agent-load-test)}"
|
||||
LF_SECRET="${LANGFUSE_RUNTIME_SECRET:-langfuse-runtime}"
|
||||
CLIENT_SECRET="${LANGFUSE_CLIENT_SECRET:-langfuse-client}"
|
||||
LF_HOST="${LANGFUSE_HOST:-$(read_env_value "$RUNTIME_ENV" LANGFUSE_HOST "http://langfuse-web.${LF_NS}.svc.cluster.local:3000")}"
|
||||
|
||||
command -v kubectl >/dev/null 2>&1 || die "kubectl not found"
|
||||
|
||||
log "Kubernetes context: $(kubectl config current-context 2>/dev/null || echo '<unknown>')"
|
||||
log "Langfuse namespace: $LF_NS"
|
||||
log "Backend namespace: $APP_NS"
|
||||
|
||||
# Do not hide kubectl errors. A credentials/context problem is not a missing Secret.
|
||||
set +e
|
||||
secret_err="$(mktemp)"
|
||||
kubectl -n "$LF_NS" get secret "$LF_SECRET" >/dev/null 2>"$secret_err"
|
||||
rc=$?
|
||||
set -e
|
||||
if (( rc != 0 )); then
|
||||
msg="$(cat "$secret_err")"
|
||||
rm -f "$secret_err"
|
||||
echo "$msg" >&2
|
||||
echo >&2
|
||||
log "Secrets visible in namespace '$LF_NS':"
|
||||
kubectl -n "$LF_NS" get secrets 2>&1 || true
|
||||
echo >&2
|
||||
log "Langfuse web Secret references:"
|
||||
kubectl -n "$LF_NS" get deployment langfuse-web \
|
||||
-o jsonpath='{.spec.template.spec.containers[0].envFrom[*].secretRef.name}{"\n"}' 2>&1 || true
|
||||
die "cannot read Secret/$LF_SECRET in namespace $LF_NS (kubectl rc=$rc). See the kubectl error above; this may be authentication/context rather than a missing Secret."
|
||||
fi
|
||||
rm -f "$secret_err"
|
||||
|
||||
get_secret(){
|
||||
local key="$1"
|
||||
local encoded
|
||||
encoded="$(kubectl -n "$LF_NS" get secret "$LF_SECRET" -o "jsonpath={.data.${key}}")"
|
||||
[[ -n "$encoded" ]] && printf '%s' "$encoded" | base64 -d
|
||||
}
|
||||
|
||||
PUBLIC_KEY="$(get_secret LANGFUSE_INIT_PROJECT_PUBLIC_KEY || true)"
|
||||
SECRET_KEY="$(get_secret LANGFUSE_INIT_PROJECT_SECRET_KEY || true)"
|
||||
[[ -n "$PUBLIC_KEY" ]] || PUBLIC_KEY="$(get_secret LANGFUSE_PUBLIC_KEY || true)"
|
||||
[[ -n "$SECRET_KEY" ]] || SECRET_KEY="$(get_secret LANGFUSE_SECRET_KEY || true)"
|
||||
|
||||
# Last-resort local fallback: the same values generated by prepare_env.py.
|
||||
if [[ -z "$PUBLIC_KEY" && -f "$LF_ENV" ]]; then
|
||||
PUBLIC_KEY="$(read_env_value "$LF_ENV" LANGFUSE_INIT_PROJECT_PUBLIC_KEY '')"
|
||||
fi
|
||||
if [[ -z "$SECRET_KEY" && -f "$LF_ENV" ]]; then
|
||||
SECRET_KEY="$(read_env_value "$LF_ENV" LANGFUSE_INIT_PROJECT_SECRET_KEY '')"
|
||||
fi
|
||||
|
||||
[[ -n "$PUBLIC_KEY" ]] || die "Langfuse public key is empty"
|
||||
[[ -n "$SECRET_KEY" ]] || die "Langfuse secret key is empty"
|
||||
|
||||
kubectl create namespace "$APP_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
kubectl -n "$APP_NS" create secret generic "$CLIENT_SECRET" \
|
||||
--from-literal=LANGFUSE_PUBLIC_KEY="$PUBLIC_KEY" \
|
||||
--from-literal=LANGFUSE_SECRET_KEY="$SECRET_KEY" \
|
||||
--from-literal=LANGFUSE_HOST="$LF_HOST" \
|
||||
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
|
||||
log "Secret/$CLIENT_SECRET synchronized into namespace $APP_NS."
|
||||
log "Public key: ${PUBLIC_KEY:0:10}... (secret key not printed)"
|
||||
103
deploy/oke/load-test/scripts/validate_dependencies.sh
Normal file
103
deploy/oke/load-test/scripts/validate_dependencies.sh
Normal file
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
ENV="$LT/.env.runtime"
|
||||
[[ -f "$ENV" ]] || { echo "Run prepare_env.sh first" >&2; exit 1; }
|
||||
source "$ENV"
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
IMAGE="$(cat "$LT/.backend-image" 2>/dev/null || true)"
|
||||
[[ -n "$IMAGE" ]] || IMAGE="${OCI_REGION_KEY}.ocir.io/${OCI_TENANCY_NAMESPACE}/${OCIR_REPOSITORY_PREFIX}/agent-template-backend:${IMAGE_TAG}"
|
||||
|
||||
"$LT/scripts/sync_langfuse_client_secret.sh"
|
||||
|
||||
kubectl -n "$NS" delete pod dependency-check --ignore-not-found >/dev/null 2>&1 || true
|
||||
cat <<YAML | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: dependency-check
|
||||
namespace: $NS
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: dependency-check
|
||||
image: $IMAGE
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: agent-backend-runtime
|
||||
env:
|
||||
- name: LANGFUSE_PUBLIC_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: langfuse-client
|
||||
key: LANGFUSE_PUBLIC_KEY
|
||||
- name: LANGFUSE_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: langfuse-client
|
||||
key: LANGFUSE_SECRET_KEY
|
||||
- name: LANGFUSE_HOST
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: langfuse-client
|
||||
key: LANGFUSE_HOST
|
||||
volumeMounts:
|
||||
- name: wallet
|
||||
mountPath: /app/wallet
|
||||
readOnly: true
|
||||
command: ["python", "-c"]
|
||||
args:
|
||||
- |
|
||||
import os, json, base64, urllib.request
|
||||
import oracledb
|
||||
from pymongo import MongoClient
|
||||
|
||||
def ok(name, detail="OK"):
|
||||
print(f"{name}: {detail}", flush=True)
|
||||
|
||||
required = ["LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_HOST"]
|
||||
missing = [k for k in required if not os.getenv(k)]
|
||||
if missing:
|
||||
raise RuntimeError("missing Langfuse variables: " + ", ".join(missing))
|
||||
|
||||
ok("wallet", os.getenv("ADB_WALLET_LOCATION", "<unset>"))
|
||||
c = oracledb.connect(
|
||||
user=os.environ["ADB_USER"], password=os.environ["ADB_PASSWORD"],
|
||||
dsn=os.environ["ADB_DSN"], config_dir=os.environ["ADB_WALLET_LOCATION"],
|
||||
wallet_location=os.environ["ADB_WALLET_LOCATION"],
|
||||
wallet_password=os.getenv("ADB_WALLET_PASSWORD"),
|
||||
)
|
||||
ok("oracle", c.version)
|
||||
c.close()
|
||||
|
||||
m = MongoClient(os.environ["MONGODB_URI"], serverSelectionTimeoutMS=10000)
|
||||
ok("mongo", str(m.admin.command("ping")))
|
||||
m.close()
|
||||
|
||||
host = os.environ["LANGFUSE_HOST"].rstrip("/")
|
||||
health = urllib.request.urlopen(host + "/api/public/health", timeout=10)
|
||||
ok("langfuse_health", str(health.status))
|
||||
|
||||
token = base64.b64encode((os.environ["LANGFUSE_PUBLIC_KEY"] + ":" + os.environ["LANGFUSE_SECRET_KEY"]).encode()).decode()
|
||||
req = urllib.request.Request(host + "/api/public/projects", headers={"Authorization": "Basic " + token})
|
||||
with urllib.request.urlopen(req, timeout=15) as r:
|
||||
body = r.read().decode("utf-8", errors="replace")
|
||||
if r.status != 200:
|
||||
raise RuntimeError(f"Langfuse authenticated API returned {r.status}")
|
||||
json.loads(body)
|
||||
ok("langfuse_authenticated_api", "200")
|
||||
|
||||
ok("dependencies", "ALL_DEPENDENCIES_OK")
|
||||
volumes:
|
||||
- name: wallet
|
||||
secret:
|
||||
secretName: agent-backend-wallet
|
||||
YAML
|
||||
|
||||
if ! kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded pod/dependency-check --timeout=180s >/dev/null 2>&1; then
|
||||
kubectl -n "$NS" describe pod dependency-check || true
|
||||
kubectl -n "$NS" logs dependency-check || true
|
||||
exit 1
|
||||
fi
|
||||
kubectl -n "$NS" logs dependency-check
|
||||
105
deploy/oke/load-test/scripts/validate_langfuse_integration.sh
Normal file
105
deploy/oke/load-test/scripts/validate_langfuse_integration.sh
Normal file
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
LT="$ROOT/deploy/oke/load-test"
|
||||
RUNTIME_ENV="$LT/.env.runtime"
|
||||
|
||||
log(){ printf '[langfuse-validate] %s\n' "$*"; }
|
||||
die(){ printf '[langfuse-validate] ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
# Do not source .env.runtime: OCI_* values can affect the OCI CLI exec plugin
|
||||
# used by kubectl. Read only the values required by this validator.
|
||||
read_env_value() {
|
||||
local file="$1" key="$2" default_value="${3:-}"
|
||||
local value=""
|
||||
if [[ -f "$file" ]]; then
|
||||
value="$(sed -n -E "s/^${key}=(.*)$/\\1/p" "$file" | tail -n 1)"
|
||||
value="${value%$'\r'}"
|
||||
if [[ "$value" == \"*\" && "$value" == *\" ]]; then value="${value:1:${#value}-2}"; fi
|
||||
if [[ "$value" == \'*\' && "$value" == *\' ]]; then value="${value:1:${#value}-2}"; fi
|
||||
fi
|
||||
printf '%s\n' "${value:-$default_value}"
|
||||
}
|
||||
|
||||
LF_NS="${LANGFUSE_NAMESPACE:-$(read_env_value "$RUNTIME_ENV" LANGFUSE_NAMESPACE langfuse)}"
|
||||
APP_NS="${K8S_NAMESPACE:-$(read_env_value "$RUNTIME_ENV" K8S_NAMESPACE agent-load-test)}"
|
||||
CLIENT_SECRET="${LANGFUSE_CLIENT_SECRET:-langfuse-client}"
|
||||
LF_HOST="${LANGFUSE_HOST:-$(read_env_value "$RUNTIME_ENV" LANGFUSE_HOST "http://langfuse-web.${LF_NS}.svc.cluster.local:3000")}"
|
||||
POD="langfuse-auth-check-$(date +%s)"
|
||||
|
||||
command -v kubectl >/dev/null 2>&1 || die "kubectl not found"
|
||||
|
||||
log "Kubernetes context: $(kubectl config current-context 2>/dev/null || echo '<unknown>')"
|
||||
|
||||
# Fail early with a Kubernetes-auth specific message.
|
||||
if ! kubectl auth can-i get secrets -n "$LF_NS" >/dev/null 2>&1; then
|
||||
die "Kubernetes authentication/authorization failed. Confirm 'kubectl get nodes' works before validating Langfuse."
|
||||
fi
|
||||
|
||||
"$LT/scripts/sync_langfuse_client_secret.sh"
|
||||
|
||||
cleanup(){
|
||||
kubectl -n "$APP_NS" delete pod "$POD" --ignore-not-found --wait=false >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Use a normal Pod manifest instead of `kubectl run --overrides`.
|
||||
# This avoids JSON-patch escaping issues and keeps secretKeyRef explicit.
|
||||
cat <<YAML | kubectl apply -f - >/dev/null
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: ${POD}
|
||||
namespace: ${APP_NS}
|
||||
labels:
|
||||
app: langfuse-auth-check
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: check
|
||||
image: docker.io/curlimages/curl:latest
|
||||
env:
|
||||
- name: LANGFUSE_HOST
|
||||
value: "${LF_HOST}"
|
||||
- name: LANGFUSE_PUBLIC_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ${CLIENT_SECRET}
|
||||
key: LANGFUSE_PUBLIC_KEY
|
||||
- name: LANGFUSE_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: ${CLIENT_SECRET}
|
||||
key: LANGFUSE_SECRET_KEY
|
||||
command: ["sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -eu
|
||||
test -n "\$LANGFUSE_PUBLIC_KEY"
|
||||
test -n "\$LANGFUSE_SECRET_KEY"
|
||||
|
||||
echo "Checking Langfuse health..."
|
||||
curl -fsS "\$LANGFUSE_HOST/api/public/health" >/dev/null
|
||||
|
||||
echo "Checking authenticated Langfuse Public API..."
|
||||
code=\$(curl -sS -o /tmp/projects.json -w "%{http_code}" \
|
||||
-u "\$LANGFUSE_PUBLIC_KEY:\$LANGFUSE_SECRET_KEY" \
|
||||
"\$LANGFUSE_HOST/api/public/projects")
|
||||
|
||||
if [ "\$code" != "200" ]; then
|
||||
echo "LANGFUSE_AUTH_FAILED HTTP \$code"
|
||||
cat /tmp/projects.json || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "LANGFUSE_AUTH_OK"
|
||||
YAML
|
||||
|
||||
if ! kubectl -n "$APP_NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$POD" --timeout=120s >/dev/null 2>&1; then
|
||||
kubectl -n "$APP_NS" logs "$POD" || true
|
||||
kubectl -n "$APP_NS" describe pod "$POD" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
kubectl -n "$APP_NS" logs "$POD"
|
||||
16
deploy/oke/load-test/scripts/watch_test.sh
Normal file
16
deploy/oke/load-test/scripts/watch_test.sh
Normal file
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
|
||||
set -a; source "$ROOT/deploy/oke/load-test/.env.runtime"; set +a
|
||||
NS="${K8S_NAMESPACE:-agent-load-test}"
|
||||
while true; do
|
||||
clear
|
||||
date
|
||||
echo '=== Pods ==='
|
||||
kubectl -n "$NS" get pods -o wide
|
||||
echo '=== HPA ==='
|
||||
kubectl -n "$NS" get hpa
|
||||
echo '=== Resource usage ==='
|
||||
kubectl -n "$NS" top pods 2>/dev/null || echo 'metrics-server not available'
|
||||
sleep 5
|
||||
done
|
||||
Reference in New Issue
Block a user