diff --git a/deploy/oke/load-test/.backend-image b/deploy/oke/load-test/.backend-image new file mode 100644 index 0000000..6b69d9c --- /dev/null +++ b/deploy/oke/load-test/.backend-image @@ -0,0 +1 @@ +gru.ocir.io/idi1o0a010nx/agent-framework-loadtest/agent-template-backend:loadtest diff --git a/deploy/oke/load-test/.env.langfuse b/deploy/oke/load-test/.env.langfuse new file mode 100644 index 0000000..d007e93 --- /dev/null +++ b/deploy/oke/load-test/.env.langfuse @@ -0,0 +1,30 @@ +# Generated by prepare_env.py. Do not commit. +CLICKHOUSE_PASSWORD=YRDQrFHp9i_g_H5orAc3-CJ19_hqBwJR +DATABASE_URL=postgresql://postgres:z8LsI4acQoIKZ_63Uqv1Lb0pLxiaMGWM@postgres.langfuse.svc.cluster.local:5432/postgres +DIRECT_URL=postgresql://postgres:z8LsI4acQoIKZ_63Uqv1Lb0pLxiaMGWM@postgres.langfuse.svc.cluster.local:5432/postgres +ENCRYPTION_KEY=1207a026e5cb136511f304e49eb6e487216b17828bfd90ab404fb4be4d442aef +LANGFUSE_CLICKHOUSE_PASSWORD=YRDQrFHp9i_g_H5orAc3-CJ19_hqBwJR +LANGFUSE_ENCRYPTION_KEY=1207a026e5cb136511f304e49eb6e487216b17828bfd90ab404fb4be4d442aef +LANGFUSE_INIT_ORG_ID=agent-framework-org +LANGFUSE_INIT_ORG_NAME=Agent Framework OCI +LANGFUSE_INIT_PROJECT_ID=agent-framework-loadtest +LANGFUSE_INIT_PROJECT_NAME=Agent Framework Load Test +LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-lf-2c82b991104f3f38f41878baecaaffd0 +LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-lf-defd21d2c6da3d4a9f301d993236958f9e56d94f9050a00f79067267b78a00a0 +LANGFUSE_INIT_USER_EMAIL=admin@loadtest.local +LANGFUSE_INIT_USER_NAME=Load Test Admin +LANGFUSE_INIT_USER_PASSWORD=HQmgpnBW3zVcG-TJMzrRgwejEUaLD7nb +LANGFUSE_MINIO_PASSWORD=Idj1YOugvY8vYhT08O60ShpLFWOVRAfy +LANGFUSE_NEXTAUTH_SECRET=54f5a8f462097696a512e81fa0141bb0739877648a63695eedd02b7475471d97 +LANGFUSE_POSTGRES_PASSWORD=z8LsI4acQoIKZ_63Uqv1Lb0pLxiaMGWM +LANGFUSE_PUBLIC_KEY=pk-lf-2c82b991104f3f38f41878baecaaffd0 +LANGFUSE_REDIS_PASSWORD=gqpWg7Ylawd3Xd-Jf496htDZyeSNWLHJ +LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY=Idj1YOugvY8vYhT08O60ShpLFWOVRAfy +LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY=Idj1YOugvY8vYhT08O60ShpLFWOVRAfy +LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY=Idj1YOugvY8vYhT08O60ShpLFWOVRAfy +LANGFUSE_SALT=8fdc487e68e7bff89f510e7cdac9c103 +LANGFUSE_SECRET_KEY=sk-lf-defd21d2c6da3d4a9f301d993236958f9e56d94f9050a00f79067267b78a00a0 +NEXTAUTH_SECRET=54f5a8f462097696a512e81fa0141bb0739877648a63695eedd02b7475471d97 +POSTGRES_PASSWORD=z8LsI4acQoIKZ_63Uqv1Lb0pLxiaMGWM +REDIS_AUTH=gqpWg7Ylawd3Xd-Jf496htDZyeSNWLHJ +SALT=8fdc487e68e7bff89f510e7cdac9c103 diff --git a/deploy/oke/load-test/.env.loadtest.example b/deploy/oke/load-test/.env.loadtest.example new file mode 100644 index 0000000..3f779ac --- /dev/null +++ b/deploy/oke/load-test/.env.loadtest.example @@ -0,0 +1,93 @@ +# ============================================================================ +# OKE architecture/load validation overlay +# This file provides DEFAULTS for keys absent from templates/agent_template_backend/.env. +# It does NOT override backend values. Use .env.loadtest for explicit overrides. +# ============================================================================ +APP_ENV=oke-load-test +LOG_LEVEL=INFO + +# OCI/OKE +K8S_NAMESPACE=agent-load-test +OKE_CLUSTER_OCID=ocid1.cluster.oc1..replace-me +OCI_CLI_PROFILE=DEFAULT +OCI_REGION=sa-saopaulo-1 +OCI_REGION_KEY=gru +OCI_TENANCY_NAMESPACE=replace-me +OCIR_REPOSITORY_PREFIX=agent-framework-loadtest +IMAGE_TAG=loadtest +OCIR_USERNAME=/ +OCIR_AUTH_TOKEN= +OCIR_EMAIL= +OCIR_PULL_SECRET_NAME=ocir-secret + + +# Existing OCI Load Balancer reuse. The Kubernetes service is NodePort; no new LB is created. +EXISTING_LB_OCID=ocid1.loadbalancer.oc1..replace-me +BACKEND_SET_NAME=agent-framework-loadtest +BACKEND_LISTENER_NAME=agent-framework-loadtest +BACKEND_LISTENER_PORT=8000 +BACKEND_LISTENER_PROTOCOL=HTTP +BACKEND_HEALTH_PATH=/health +BACKEND_NODE_PORT=32116 +LOADTEST_TARGET_MODE=external +# Optional: set this to bypass OCI CLI IP discovery. +# LOADTEST_EXTERNAL_URL=http://203.0.113.10:8000 + +# OCI GenAI. For OKE prefer workload identity or instance principal when available. +# Keep oci_sdk for the real end-to-end profile. +LLM_PROVIDER=oci_sdk +OCI_AUTH_MODE=oke_workload_identity + +# Shared persistence: required for horizontal-scale validation. +SESSION_REPOSITORY_PROVIDER=autonomous +MEMORY_REPOSITORY_PROVIDER=autonomous +CHECKPOINT_REPOSITORY_PROVIDER=autonomous +USAGE_REPOSITORY_PROVIDER=autonomous +VECTOR_STORE_PROVIDER=autonomous +GRAPH_STORE_PROVIDER=autonomous + +# Wallet is mounted by Kubernetes Secret at this fixed in-container path. +ADB_WALLET_LOCATION=/app/wallet + +# Mongo-compatible endpoint / sequence store. Use the private VCN endpoint. +# IMPORTANT: standard port restored to 27017 (not 37017). +# Example only; replace host/user/password with the Autonomous/Mongo endpoint used in OCI. +MONGODB_URI=mongodb://user:password@autonomous-private-endpoint:27017/?tls=true +MONGODB_DATABASE=agent_platform +PUBSUB_SEQUENCE_PROVIDER=mongodb +PUBSUB_SEQUENCE_MONGODB_URI=mongodb://user:password@autonomous-private-endpoint:27017/?tls=true +PUBSUB_SEQUENCE_MONGODB_DATABASE=agent_platform +PUBSUB_SEQUENCE_MEMORY_FALLBACK=false + +# Langfuse is installed in the same OKE cluster by deploy_langfuse.sh. +ENABLE_LANGFUSE=true +LANGFUSE_HOST=http://langfuse-web.langfuse.svc.cluster.local:3000 +LANGFUSE_TRACE_MODE=compact +LANGFUSE_IGNORE_HEALTHCHECKS=true +LANGFUSE_IGNORED_PATHS=/health,/ready,/metrics + +# Keep analytics disabled unless Pub/Sub is part of the test objective. +ENABLE_ANALYTICS=false + +# MCP can be enabled in a second pass if you also deploy mcp-gateway. +ENABLE_MCP_TOOLS=false +MCP_GATEWAY_ENABLED=false + +# Load-test deployment sizing +LOADTEST_MIN_REPLICAS=4 +LOADTEST_MAX_REPLICAS=30 +LOADTEST_CPU_REQUEST=500m +LOADTEST_CPU_LIMIT=2000m +LOADTEST_MEMORY_REQUEST=768Mi +LOADTEST_MEMORY_LIMIT=2Gi + +# Load generator defaults +LOADTEST_TARGET=http://agent-template-backend-lb.agent-load-test.svc.cluster.local:8000 +LOADTEST_AGENT_ID=telecom_contas +LOADTEST_TENANT_ID=loadtest +LOADTEST_DURATION=10m +LOADTEST_VUS=250 +LOADTEST_MAX_VUS=1000 +LOADTEST_RPS=100 +LOADTEST_SCENARIO=unique_sessions +LOADTEST_MESSAGE=Explique em uma frase o que é uma fatura de telecom. diff --git a/deploy/oke/load-test/.env.loadtest.override.example b/deploy/oke/load-test/.env.loadtest.override.example new file mode 100644 index 0000000..eb0c4d0 --- /dev/null +++ b/deploy/oke/load-test/.env.loadtest.override.example @@ -0,0 +1,13 @@ +# Optional explicit overrides for OKE/load testing. +# Copy to .env.loadtest ONLY when you intentionally want to replace values +# inherited from templates/agent_template_backend/.env. +# +# cp deploy/oke/load-test/.env.loadtest.override.example \ +# deploy/oke/load-test/.env.loadtest + +# Example: +# OCI_REGION=sa-saopaulo-1 +# OCI_AUTH_MODE=oke_workload_identity +# ENABLE_ANALYTICS=true +# ENABLE_MCP_TOOLS=true +# MONGODB_URI=mongodb://... diff --git a/deploy/oke/load-test/.env.runtime b/deploy/oke/load-test/.env.runtime new file mode 100644 index 0000000..79fc8ad --- /dev/null +++ b/deploy/oke/load-test/.env.runtime @@ -0,0 +1,159 @@ +# Generated by prepare_env.py. Do not commit. +ADB_DSN=oradb23ai_high +ADB_PASSWORD=Moniquinha19721972 +ADB_TABLE_PREFIX=AGENTFW +ADB_USER=admin +ADB_WALLET_LOCATION=/app/wallet +ADB_WALLET_PASSWORD=Moniquinha1972 +AGENT_PUBSUB_TOPIC=teste_pubsub +ANALYTICS_PROVIDERS=pubsub +API_HOST=0.0.0.0 +API_PORT=8000 +APP_ENV=oke-load-test +APP_NAME=ai-agent-template +CHECKPOINT_REPOSITORY_PROVIDER=autonomous +CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 +DEFAULT_CHANNEL=web +EMBEDDING_PROVIDER=oci +ENABLE_ANALYTICS=false +ENABLE_CONVERSATION_SUMMARY_MEMORY=true +ENABLE_INPUT_GUARDRAILS=true +ENABLE_JUDGES=true +ENABLE_LANGFUSE=true +ENABLE_LANGFUSE_ANALYTICS_PUBLISHER=false +ENABLE_LANGFUSE_OPENAI_AUTO_INSTRUMENTATION=true +ENABLE_LLM_ROUTER=true +ENABLE_LONG_TERM_MEMORY=true +ENABLE_MCP_TOOLS=false +ENABLE_OCI_STREAMING=false +ENABLE_OTEL=false +ENABLE_OUTPUT_GUARDRAILS=true +ENABLE_OUTPUT_SUPERVISOR=true +ENABLE_PARALLEL_GUARDRAILS=true +ENABLE_REDIS_CACHE=false +ENABLE_ROUTE_STICKINESS=true +ENABLE_SUPERVISOR=true +ENABLE_TEXT_ADAPTER=true +ENABLE_VOICE_ADAPTER=true +ENABLE_WHATSAPP_ADAPTER=true +END_SESSION_MESSAGE='Atendimento encerrado. Obrigado pelo contato.' +FRAMEWORK_CHANNEL_INPUT_MODE=embedded +GCP_PROJECT_ID=project-7f829b68-bf43-442c-ad3 +GCP_PUBSUB_TIMEOUT_SECONDS=30 +GCP_PUBSUB_TOPIC=teste_pubsub +GCP_PUBSUB_TOPIC_PATH=projects/project-7f829b68-bf43-442c-ad3/topics/teste_pubsub +GOOGLE_APPLICATION_CREDENTIALS=/mnt/d/Dropbox/ORACLE/TIM/FY27/Testes/sa-hoshikawa.json +GRAPH_STORE_PROVIDER=autonomous +GUARDRAILS_CONFIG_PATH=./config/guardrails.yaml +GUARDRAILS_FAIL_FAST=true +HUMAN_HANDOFF_MESSAGE='Vou encaminhar seu atendimento para uma pessoa.' +IDENTITY_CONFIG_PATH=./config/identity.yaml +IMAGE_TAG=loadtest +JUDGES_CONFIG_PATH=./config/judges.yaml +K8S_NAMESPACE=agent-load-test +LANGFUSE_COMPACT_SUPPRESSED_PREFIXES=llm.chat_completion +LANGFUSE_COMPACT_VISIBLE_EVENT_PREFIXES='AGA.,NOC., IC.' +LANGFUSE_HOST=http://langfuse-web.langfuse.svc.cluster.local:3000 +LANGFUSE_IGNORED_PATHS=/health,/ready,/metrics +LANGFUSE_IGNORE_HEALTHCHECKS=true +LANGFUSE_PUBLIC_KEY=pk-lf-2c82b991104f3f38f41878baecaaffd0 +LANGFUSE_SECRET_KEY=sk-lf-defd21d2c6da3d4a9f301d993236958f9e56d94f9050a00f79067267b78a00a0 +LANGFUSE_TRACE_MODE=compact +LLM_MAX_TOKENS=2048 +LLM_PROVIDER=oci_sdk +LLM_TEMPERATURE=0.2 +LLM_TIMEOUT_SECONDS=120 +LOADTEST_AGENT_ID=telecom_contas +LOADTEST_CPU_LIMIT=2000m +LOADTEST_CPU_REQUEST=500m +LOADTEST_DURATION=10m +LOADTEST_MAX_REPLICAS=30 +LOADTEST_MAX_VUS=1000 +LOADTEST_MEMORY_LIMIT=2Gi +LOADTEST_MEMORY_REQUEST=768Mi +LOADTEST_MESSAGE='Explique em uma frase o que é uma fatura de telecom.' +LOADTEST_MIN_REPLICAS=4 +LOADTEST_RPS=100 +LOADTEST_SCENARIO=unique_sessions +LOADTEST_TARGET=http://agent-template-backend-lb.agent-load-test.svc.cluster.local:8000 +LOADTEST_TENANT_ID=loadtest +LOADTEST_VUS=250 +LOG_LEVEL=INFO +LONG_TERM_MEMORY_AUTO_EXTRACT=true +LONG_TERM_MEMORY_INJECT_CONTEXT=true +LONG_TERM_MEMORY_MAX_CONTEXT_ITEMS=20 +LONG_TERM_MEMORY_MIN_CONFIDENCE=0.70 +LONG_TERM_MEMORY_PROVIDER=sqlite +LONG_TERM_MEMORY_SQLITE_PATH=./data/agent_framework.db +LONG_TERM_MEMORY_TABLE=agentfw_long_term_memory +MCP_GATEWAY_ENABLED=false +MCP_PARAMETER_MAPPING_PATH=./config/mcp_parameter_mapping.yaml +MCP_SERVERS_CONFIG_PATH=./config/mcp_servers.yaml +MCP_TOOL_TIMEOUT_SECONDS=30 +MEMORY_CONTEXT_STRATEGY=summary +MEMORY_HISTORY_LIMIT=80 +MEMORY_INJECT_RECENT_MESSAGES=true +MEMORY_INJECT_SUMMARY=true +MEMORY_MAX_SUMMARY_CHARS=6000 +MEMORY_RECENT_MESSAGES_LIMIT=8 +MEMORY_REPOSITORY_PROVIDER=autonomous +MEMORY_SUMMARY_TRIGGER_MESSAGES=20 +MEMORY_SUMMARY_USE_LLM=true +MONGODB_DATABASE=agent_platform +MONGODB_URI='mongodb://user:password@autonomous-private-endpoint:27017/?tls=true' +OCIR_REPOSITORY_PREFIX=agent-framework-loadtest +OCI_AUTH_MODE=oke_workload_identity +OCI_CLI_PROFILE=LATINOAMERICA-SaoPaulo +OCI_COMPARTMENT_ID=ocid1.compartment.oc1..aaaaaaaaexpiw4a7dio64mkfv2t273s2hgdl6mgfvvyv7tycalnjlvpvfl3q +OCI_CONFIG_FILE='~/.oci/config' +OCI_EMBEDDING_MODEL=cohere.embed-multilingual-v3.0 +OCI_GENAI_API_KEY=sk-ph3FgX6iP3fxAQCXb9IpPIDTadkeeYAWntUWhzcWysIM6zsS +OCI_GENAI_BASE_URL=https://inference.generativeai.us-chicago-1.oci.oraclecloud.com +OCI_GENAI_MODEL=openai.gpt-4.1 +OCI_GENAI_PROJECT_OCID='' +OCI_PROFILE=LATINOAMERICA-Chicago +OCI_REGION=sa-saopaulo-1 +OCI_REGION_KEY=gru +OCIR_USERNAME=idi1o0a010nx/oracleidentitycloudservice/cristiano.hoshikawa@oracle.com +OCIR_AUTH_TOKEN="A5XBm>8gvA2oDy;it1lq" +OCIR_EMAIL=cristiano.hoshikawa@oracle.com +OCIR_PULL_SECRET_NAME=ocir-secret +OCI_STREAM_ENDPOINT='' +OCI_STREAM_OCID='' +OCI_STREAM_PARTITION_KEY=agent-events +OCI_TENANCY_NAMESPACE=idi1o0a010nx +OKE_CLUSTER_OCID=ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaabpismmmj6kwo4idomzso5fr32xlnbjcm74pfa7bfhc52plqe64oq +EXISTING_LB_OCID=ocid1.loadbalancer.oc1.sa-saopaulo-1.aaaaaaaaklbmmriyabvxradqkchy5cobelzxj37rn27p2rsfef5uez54jfqa +BACKEND_SET_NAME=agent-framework-loadtest +BACKEND_LISTENER_NAME=agent-framework-loadtest +BACKEND_LISTENER_PORT=8000 +BACKEND_LISTENER_PROTOCOL=HTTP +BACKEND_HEALTH_PATH=/health +OTEL_EXPORTER_OTLP_ENDPOINT='' +OTEL_SERVICE_NAME=ai-agent-template +OUTPUT_SUPERVISOR_MAX_RETRIES=3 +PROJECT_ID=project-7f829b68-bf43-442c-ad3 +PROMPT_POLICY_PATH=./config/prompt_policy.yaml +PUBSUB_SEQUENCE_ENABLED=true +PUBSUB_SEQUENCE_KEY_PREFIX=observer:sequence +PUBSUB_SEQUENCE_MEMORY_FALLBACK=false +PUBSUB_SEQUENCE_MONGODB_COLLECTION=observer_event_counters +PUBSUB_SEQUENCE_MONGODB_DATABASE=agent_platform +PUBSUB_SEQUENCE_MONGODB_URI='mongodb://user:password@autonomous-private-endpoint:27017/?tls=true' +PUBSUB_SEQUENCE_PROVIDER=mongodb +PUBSUB_SEQUENCE_TTL_SECONDS=86400 +PUBSUB_TOPIC_PATH=projects/project-7f829b68-bf43-442c-ad3/topics/teste_pubsub +RAG_FILE_GLOBS='*.md,*.txt,*.yaml,*.yml,*.json' +RAG_TOP_K=5 +REDIS_URL=redis://localhost:6379/0 +ROUTE_STICKINESS_CONFIDENCE_THRESHOLD=0.90 +ROUTE_STICKINESS_HISTORY_TURNS=2 +ROUTE_STICKINESS_LLM_PROFILE=route_continuity +ROUTE_STICKINESS_MAX_TOKENS=80 +ROUTING_CONFIG_PATH=./config/routing.yaml +ROUTING_MODE=router +SESSION_REPOSITORY_PROVIDER=autonomous +SIMULATE_AGA_MISSING_TRANSACTION_ID=true +TOOLS_CONFIG_PATH=./config/tools.yaml +USAGE_REPOSITORY_PROVIDER=autonomous +VECTOR_STORE_PROVIDER=autonomous diff --git a/deploy/oke/load-test/CHANGES_2026-08-17.md b/deploy/oke/load-test/CHANGES_2026-08-17.md new file mode 100644 index 0000000..c2eb8ae --- /dev/null +++ b/deploy/oke/load-test/CHANGES_2026-08-17.md @@ -0,0 +1,36 @@ +# Alterações do pacote OKE load test — 2026-08-17 + +- substituído o fluxo de criação de novo OCI Load Balancer pelo reaproveitamento de `EXISTING_LB_OCID`; +- Service externo alterado para `NodePort`, com `BACKEND_NODE_PORT` configurável; +- `configure_existing_lb.sh` cria/reutiliza backend set, backends dos workers e listener; +- smoke/load test externo resolvem o IP do LB existente ou usam `LOADTEST_EXTERNAL_URL`; +- bootstrap headless do Langfuse passa a ser validado com API autenticada; +- criado `sync_langfuse_client_secret.sh` para copiar as keys do projeto Langfuse para o namespace do backend; +- backend recebe `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY` e `LANGFUSE_HOST` explicitamente do Secret `langfuse-client`; +- checksum de runtime no pod template força rollout quando env/keys mudam; +- `validate_dependencies.sh` agora valida Oracle, Mongo, health do Langfuse e API autenticada; +- smoke test confirma ingestão do trace no Langfuse (`LANGFUSE_TRACE_OK`); +- `01_deploy_all.sh` executa configuração do LB existente antes das validações; +- manual reescrito com fluxo completo, troubleshooting e critérios de aprovação. + +## v4 - Langfuse Secret synchronization fix + +- `sync_langfuse_client_secret.sh` no longer sources `.env.runtime`, preventing backend OCI variables from altering the OCI CLI exec-plugin environment used by `kubectl`. +- A failed `kubectl get secret` is no longer mislabeled as `Secret not found`; the actual Kubernetes authentication/context error is printed. +- `deploy_langfuse.sh` verifies `Secret/langfuse-runtime` immediately before synchronization and explicitly passes Langfuse/backend namespaces to the child script. +- Synchronization can fall back to `.env.langfuse` only for key values after Kubernetes Secret access has been validated. + +## Documentação — inicialização manual do Langfuse + +- README atualizado para documentar explicitamente o fluxo pela UI após subir o Langfuse. +- Passos incluídos: criação de usuário, organização/empresa, projeto e API keys. +- Documentado onde copiar `pk-lf-*` e `sk-lf-*` em `.env.langfuse` e `.env.runtime`. +- Incluída atualização de `Secret/langfuse-runtime`, sincronização para `Secret/langfuse-client` e validação obrigatória com `LANGFUSE_AUTH_OK` antes do deploy do backend. +- O bootstrap headless continua existente no código, mas o procedimento operacional recomendado para esta validação passa a confirmar/criar o projeto e as chaves pela UI. + + +## v6 - Langfuse validation fix +- Replaced `kubectl run --overrides` JSON with a regular Pod YAML manifest. +- Fixed `Invalid JSON Patch` caused by invalid JSON escaping in the validator. +- Validator no longer sources `.env.runtime`; it reads only required keys. +- Added Kubernetes authentication preflight and explicit Langfuse HTTP auth result. diff --git a/deploy/oke/load-test/README.md b/deploy/oke/load-test/README.md new file mode 100644 index 0000000..85eb70b --- /dev/null +++ b/deploy/oke/load-test/README.md @@ -0,0 +1,728 @@ +# Validação arquitetural de carga — Agent Framework OCI / OKE + +Este pacote executa validação arquitetural e teste de carga do `agent_template_backend` em OCI OKE usando **persistência compartilhada**, **OCI Generative AI**, **Langfuse v3** e um **OCI Load Balancer preexistente**. O pacote não cria um novo OCI Load Balancer: o acesso externo é feito por `NodePort` nos workers e por um listener/backend set criado no LB informado em `EXISTING_LB_OCID`. + +## Arquitetura do teste + +```text +Cliente / k6 externo + | + v +OCI Load Balancer existente :BACKEND_LISTENER_PORT + | + v +backend set agent-framework-loadtest + | + +--> worker-1:BACKEND_NODE_PORT + +--> worker-2:BACKEND_NODE_PORT + +--> worker-N:BACKEND_NODE_PORT + | + v +Kubernetes Service NodePort + | + v +agent-template-backend (HPA) + | + +--> Autonomous Database / wallet + +--> Mongo-compatible endpoint + +--> OCI Generative AI + +--> Langfuse v3 +``` + +O Service `agent-template-backend` continua `ClusterIP` para testes internos. O Service `agent-template-backend-lb` é `NodePort`; ele **não** solicita um novo LB à OCI. + +## Estrutura principal + +```text +deploy/oke/load-test/ + .env.loadtest.example + .env.runtime # gerado; não commitar + .env.langfuse # gerado; não commitar + README.md + k8s/ + agent-backend.yaml + k6-job.yaml + langfuse/ + langfuse-k8s.yaml + loadgen/ + loadtest.js + scripts/ + prepare_env.sh + configure_kubeconfig.sh + deploy_langfuse.sh + sync_langfuse_client_secret.sh + validate_langfuse_integration.sh + build_push_backend.sh + deploy_backend.sh + configure_existing_lb.sh + validate_dependencies.sh + smoke_test.sh + run_load_test.sh + run_architecture_suite.sh + watch_test.sh + collect_results.sh +``` + +--- + +# 1. Preparar `.env.runtime` + +Execute da raiz do repositório: + +```bash +./deploy/oke/load-test/scripts/prepare_env.sh +``` + +O script usa `templates/agent_template_backend/.env` como configuração principal, completa somente as chaves ausentes com `deploy/oke/load-test/.env.loadtest.example` e gera: + +```text +deploy/oke/load-test/.env.runtime +deploy/oke/load-test/.env.langfuse +``` + +> **Importante sobre Langfuse:** o script pode gerar valores de inicialização (`LANGFUSE_INIT_*`) para bootstrap headless, mas, para esta validação, o procedimento recomendado é confirmar o ambiente pela UI do Langfuse e criar manualmente o usuário, organização/empresa, projeto e API keys. As chaves criadas na UI devem ser copiadas para os arquivos de configuração antes do deploy do backend. + +Preencha no `.env.runtime` os parâmetros reais de OCI/DB/OCIR e o LB existente. Para o LB: + +```bash +EXISTING_LB_OCID=ocid1.loadbalancer.oc1.sa-saopaulo-1.... +BACKEND_SET_NAME=agent-framework-loadtest +BACKEND_LISTENER_NAME=agent-framework-loadtest +BACKEND_LISTENER_PORT=8000 +BACKEND_LISTENER_PROTOCOL=HTTP +BACKEND_HEALTH_PATH=/health +BACKEND_NODE_PORT=32116 +LOADTEST_TARGET_MODE=external +``` + +Opcionalmente, fixe diretamente a URL externa: + +```bash +LOADTEST_EXTERNAL_URL=http://:8000 +``` + +--- + +# 2. Wallet e acesso ao OKE + +Copie a wallet real para: + +```text +templates/agent_template_backend/wallet/ +``` + +Ela será criada como Kubernetes Secret e montada em `/app/wallet`. + +Configure kubeconfig: + +```bash +./deploy/oke/load-test/scripts/configure_kubeconfig.sh +kubectl get nodes -o wide +kubectl top nodes +``` + +`kubectl top nodes` deve funcionar para o HPA baseado em CPU/memória. + +--- + +# 3. Subir o Langfuse v3 e criar manualmente usuário, organização, projeto e API keys + +## 3.1 Subir a infraestrutura do Langfuse + +Execute: + +```bash +./deploy/oke/load-test/scripts/deploy_langfuse.sh +``` + +O deploy cria os componentes Kubernetes do Langfuse v3, incluindo web, worker, Postgres, Redis, ClickHouse, MinIO e os Secrets de runtime. + +Se a etapa de validação autenticada falhar porque ainda não existem API keys válidas, prossiga com a inicialização manual abaixo. A infraestrutura do Langfuse já pode estar operacional mesmo que a validação das chaves ainda não tenha passado. + +Confirme: + +```bash +kubectl -n langfuse get pods +kubectl -n langfuse get svc langfuse-web +``` + +Os pods principais devem estar `Running`/`Ready`. + +## 3.2 Abrir a interface do Langfuse + +Abra um port-forward em outro terminal: + +```bash +kubectl -n langfuse port-forward --address 127.0.0.1 svc/langfuse-web 3005:3000 +``` + +Acesse no navegador: + +```text +http://127.0.0.1:3005 +``` + +## 3.3 Criar o usuário + +Na primeira abertura do Langfuse: + +1. crie o usuário administrador; +2. faça login com esse usuário; +3. confirme que a interface principal do Langfuse foi carregada. + +Para ambiente de teste, pode ser utilizado um usuário dedicado ao load test. Não use credenciais pessoais de produção no arquivo do projeto. + +## 3.4 Criar a organização/empresa + +Dentro do Langfuse, crie ou selecione a organização que será usada pelo teste. + +Sugestão de nome: + +```text +Agent Framework OCI +``` + +A UI do Langfuse pode usar o termo **Organization**. Neste manual, organização/empresa representam a mesma entidade de agrupamento do projeto. + +## 3.5 Criar o projeto + +Dentro dessa organização, crie o projeto que receberá os traces do teste. + +Sugestão: + +```text +Agent Framework Load Test +``` + +Não prossiga para o backend enquanto não conseguir abrir esse projeto na UI. + +## 3.6 Criar as API keys do projeto + +No projeto criado, abra a área de configuração/API Keys e crie um novo par de credenciais. + +Você receberá duas chaves: + +```text +Public Key: pk-lf-... +Secret Key: sk-lf-... +``` + +> **Atenção:** copie a `Secret Key` no momento da criação. Dependendo da tela/versão, ela pode não ser exibida novamente. + +Essas são as credenciais que o `agent_template_backend` usará para enviar traces ao Langfuse. + +## 3.7 Copiar as API keys para `.env.langfuse` + +Edite: + +```text +deploy/oke/load-test/.env.langfuse +``` + +Substitua/ajuste estas quatro linhas com o par criado na UI: + +```bash +LANGFUSE_PUBLIC_KEY=pk-lf-COLE_A_PUBLIC_KEY_AQUI +LANGFUSE_SECRET_KEY=sk-lf-COLE_A_SECRET_KEY_AQUI +LANGFUSE_INIT_PROJECT_PUBLIC_KEY=pk-lf-COLE_A_MESMA_PUBLIC_KEY_AQUI +LANGFUSE_INIT_PROJECT_SECRET_KEY=sk-lf-COLE_A_MESMA_SECRET_KEY_AQUI +``` + +Para um projeto já criado manualmente na UI, os campos `LANGFUSE_INIT_PROJECT_*` são mantidos com os mesmos valores para que os scripts de sincronização do pacote encontrem um único par de chaves consistente. + +Não exiba nem versione a `LANGFUSE_SECRET_KEY`. + +## 3.8 Copiar as API keys para `.env.runtime` + +Edite também: + +```text +deploy/oke/load-test/.env.runtime +``` + +Garanta: + +```bash +ENABLE_LANGFUSE=true +LANGFUSE_HOST=http://langfuse-web.langfuse.svc.cluster.local:3000 +LANGFUSE_PUBLIC_KEY=pk-lf-COLE_A_PUBLIC_KEY_AQUI +LANGFUSE_SECRET_KEY=sk-lf-COLE_A_SECRET_KEY_AQUI +``` + +As chaves em `.env.runtime` devem ser exatamente as mesmas criadas no projeto pela UI. + +## 3.9 Atualizar o Secret Kubernetes do Langfuse + +Depois de colar as chaves nos arquivos, atualize `Secret/langfuse-runtime` sem imprimir os valores: + +```bash +set -a +source deploy/oke/load-test/.env.langfuse +set +a + +kubectl -n langfuse create secret generic langfuse-runtime \ + --from-env-file=deploy/oke/load-test/.env.langfuse \ + --dry-run=client -o yaml | kubectl apply -f - +``` + +Para um projeto criado manualmente, não é necessário recriar o banco do Langfuse. O objetivo desta etapa é manter o Secret usado pelos scripts sincronizado com as credenciais reais do projeto. + +## 3.10 Sincronizar as chaves para o namespace do backend + +Execute: + +```bash +LANGFUSE_NAMESPACE=langfuse \ +K8S_NAMESPACE=agent-load-test \ +./deploy/oke/load-test/scripts/sync_langfuse_client_secret.sh +``` + +O script cria/atualiza: + +```text +namespace: agent-load-test +Secret: langfuse-client +``` + +com: + +```text +LANGFUSE_PUBLIC_KEY +LANGFUSE_SECRET_KEY +LANGFUSE_HOST +``` + +Confira apenas a existência do Secret: + +```bash +kubectl -n agent-load-test get secret langfuse-client +``` + +Para conferir somente a Public Key: + +```bash +kubectl -n agent-load-test get secret langfuse-client \ + -o jsonpath='{.data.LANGFUSE_PUBLIC_KEY}' | base64 -d + +echo +``` + +Ela deve começar com: + +```text +pk-lf- +``` + +## 3.11 Validar autenticação no Langfuse antes de prosseguir + +Execute: + +```bash +./deploy/oke/load-test/scripts/validate_langfuse_integration.sh +``` + +Resultado esperado: + +```text +LANGFUSE_AUTH_OK +``` + +Esse teste chama o Langfuse pela rede interna do OKE e usa `LANGFUSE_PUBLIC_KEY` + `LANGFUSE_SECRET_KEY`. Portanto, `LANGFUSE_AUTH_OK` comprova que o par de chaves é aceito pelo projeto. + +Se receber `401`/`403`, não prossiga para o backend. Revise: + +```text +.env.langfuse +.env.runtime +Secret/langfuse-runtime no namespace langfuse +Secret/langfuse-client no namespace agent-load-test +``` + +## 3.12 Ordem obrigatória antes de continuar + +A sequência esperada é: + +```text +Langfuse Kubernetes operacional + ↓ +criar usuário + ↓ +criar/selecionar organização (empresa) + ↓ +criar projeto + ↓ +criar Public Key + Secret Key + ↓ +colar keys em .env.langfuse + ↓ +colar keys em .env.runtime + ↓ +atualizar Secret/langfuse-runtime + ↓ +sync_langfuse_client_secret.sh + ↓ +validate_langfuse_integration.sh + ↓ +LANGFUSE_AUTH_OK + ↓ +somente então fazer deploy do backend +``` + +--- + +# 4. Build e push do backend + +Garanta que o repositório OCIR já exista e faça login: + +```bash +docker login gru.ocir.io +./deploy/oke/load-test/scripts/build_push_backend.sh +``` + +A imagem utilizada fica registrada em `deploy/oke/load-test/.backend-image`. + +--- + +# 5. Deploy do backend com NodePort + +Execute: + +```bash +./deploy/oke/load-test/scripts/deploy_backend.sh +``` + +O deploy cria/atualiza: + +- `agent-backend-runtime`; +- `agent-backend-wallet`; +- `langfuse-client`; +- Deployment do backend; +- `ClusterIP` interno; +- `NodePort` externo (`BACKEND_NODE_PORT`, default `32116`); +- HPA; +- PDB. + +Confira: + +```bash +kubectl -n agent-load-test get pods -o wide +kubectl -n agent-load-test get svc +kubectl -n agent-load-test get hpa +``` + +O Service externo deve aparecer como `NodePort`, e não `LoadBalancer`. + +--- + +# 6. Reutilizar o OCI Load Balancer existente + +Depois que o NodePort existir: + +```bash +./deploy/oke/load-test/scripts/configure_existing_lb.sh +``` + +O script é idempotente. Ele: + +1. valida `EXISTING_LB_OCID`; +2. descobre `BACKEND_NODE_PORT` no Service; +3. verifica o backend set; +4. cria `BACKEND_SET_NAME` se não existir; +5. aguarda a operação OCI concluir; +6. descobre os `InternalIP` dos workers OKE; +7. adiciona os workers ainda não cadastrados como `:`; +8. verifica/cria o listener `BACKEND_LISTENER_NAME`; +9. aponta o listener para o backend set; +10. mostra o health do backend set. + +Arquitetura resultante: + +```text +OCI LB existente + -> listener :8000 + -> backend set + -> worker InternalIP:32116 + -> Service NodePort + -> pods do backend +``` + +As Security Lists/NSGs precisam permitir tráfego do LB para os workers na porta NodePort. + +--- + +# 7. Validar dependências antes do smoke/carga + +Execute: + +```bash +./deploy/oke/load-test/scripts/validate_dependencies.sh +``` + +Esse teste é executado **de dentro do OKE** usando a mesma imagem, env e wallet do backend. Ele valida: + +- conexão real com Oracle/Autonomous usando `/app/wallet`; +- `ping` real no Mongo-compatible endpoint; +- `/api/public/health` do Langfuse; +- presença das API keys Langfuse; +- autenticação real em `/api/public/projects` usando `LANGFUSE_PUBLIC_KEY` + `LANGFUSE_SECRET_KEY`. + +Resultado esperado termina com: + +```text +dependencies: ALL_DEPENDENCIES_OK +``` + +Esse passo valida conectividade e autenticação. Ele não faz uma chamada de negócio ao LLM; isso é responsabilidade do smoke test. + +--- + +# 8. Smoke test end-to-end + +## Externo — caminho real pelo OCI LB existente + +```bash +./deploy/oke/load-test/scripts/smoke_test.sh external +``` + +Caminho validado: + +```text +cliente + -> OCI LB existente + -> listener + -> backend set + -> NodePort + -> pod + -> LangGraph/framework + -> RAG/memória conforme configuração + -> OCI Generative AI (quando LLM_PROVIDER=oci_sdk) + -> persistência + -> Langfuse +``` + +O smoke executa `/health`, depois `POST /gateway/message`. Quando Langfuse está habilitado, ele também consulta a Public API autenticada e aguarda o trace da sessão `smoke-*`. O sucesso final inclui: + +```text +LANGFUSE_TRACE_OK +``` + +Assim, um `HTTP 200` sozinho não é mais considerado evidência suficiente de observabilidade. + +## Interno — sem OCI LB + +```bash +./deploy/oke/load-test/scripts/smoke_test.sh internal +``` + +Esse modo usa o `ClusterIP` e serve para separar problemas do backend/Kubernetes de problemas do LB externo. + +Para provar chamadas reais de OCI Generative AI, mantenha: + +```text +LLM_PROVIDER=oci_sdk +OCI_AUTH_MODE=oke_workload_identity +``` + +Nos logs devem aparecer `OCI SDK GenAI client`, `OnDemandServingMode` e eventos `llm.*` com o modelo configurado. + +--- + +# 9. Teste de carga + +## Interno + +```bash +./deploy/oke/load-test/scripts/run_load_test.sh internal +``` + +Alvo: + +```text +http://agent-template-backend.agent-load-test.svc.cluster.local:8000 +``` + +## Externo pelo LB preexistente + +```bash +./deploy/oke/load-test/scripts/run_load_test.sh external +``` + +O script resolve o endereço do LB usando `EXISTING_LB_OCID` via OCI CLI, ou usa `LOADTEST_EXTERNAL_URL` se definida. + +Acompanhe: + +```bash +kubectl -n agent-load-test logs -f job/agent-load-generator +./deploy/oke/load-test/scripts/watch_test.sh +``` + +Distribuição pelo LB: + +```bash +./deploy/oke/load-test/scripts/check_load_balancing.sh +``` + +--- + +# 10. Suite arquitetural + +Externa: + +```bash +./deploy/oke/load-test/scripts/run_architecture_suite.sh external +``` + +Interna: + +```bash +./deploy/oke/load-test/scripts/run_architecture_suite.sh internal +``` + +Cenários padrão: + +```text +warmup 5 rps 2 min unique_sessions +baseline 25 rps 5 min unique_sessions +scale 100 rps 10 min unique_sessions +shared-state 50 rps 10 min shared_sessions +``` + +`unique_sessions` estressa criação de estado/checkpoints/DB/Langfuse/LLM. `shared_sessions` reutiliza sessões entre chamadas e é importante para provar que a aplicação não depende de memória local do pod. + +--- + +# 11. Execução completa automatizada + +Depois de preencher `.env.runtime` e colocar a wallet: + +```bash +./deploy/oke/load-test/scripts/01_deploy_all.sh +``` + +Ordem: + +```text +Langfuse + API keys + -> build/push backend + -> deploy backend NodePort + -> configurar LB existente + -> validar dependências + -> smoke externo + confirmação do trace Langfuse +``` + +Só depois desse fluxo passar execute carga sustentada. + +--- + +# 12. Critérios de aprovação + +Antes da carga: + +```text +kubectl nodes Ready +metrics-server OK +Langfuse AUTH OK +Oracle connection OK +Mongo ping OK +backend rollout OK +OCI LB backend-set healthy +smoke HTTP 200 +OCI GenAI real nos logs (perfil end-to-end) +LANGFUSE_TRACE_OK +``` + +Durante a carga, procure: + +- nenhum `CrashLoopBackOff`/`OOMKilled`; +- ausência de deadlock/cross-event-loop errors; +- sessão consistente entre pods; +- checkpoints recuperáveis por qualquer réplica; +- HPA escalando quando as métricas atingirem os targets; +- 5xx próximos de zero; +- 429/timeouts do LLM separados de erros internos; +- traces chegando ao Langfuse sem criar falha em cascata. + +O backend é I/O-bound em chamadas de LLM/DB; CPU/memória não são métricas perfeitas de autoscaling. Para produção, considere métricas de aplicação como `in_flight_requests`, `request_queue_depth` e p95 de latência via KEDA/Prometheus Adapter. + +--- + +# 13. Coletar evidências + +```bash +./deploy/oke/load-test/scripts/collect_results.sh +``` + +Os arquivos ficam em `deploy/oke/load-test/results/`. Use em conjunto com Langfuse para correlacionar request/session/trace. + +--- + +# 14. Resiliência + +Com carga ativa: + +```bash +kubectl -n agent-load-test delete pod +kubectl -n agent-load-test rollout restart deployment/agent-template-backend +kubectl -n agent-load-test rollout status deployment/agent-template-backend +``` + +A perda de uma réplica não deve perder estado compartilhado nem interromper o serviço enquanto houver capacidade saudável. + +--- + +# 15. Troubleshooting de credenciais Langfuse em instalação já existente + +O bootstrap headless é idempotente e foi desenhado para criar recursos que ainda não existem. Em um ambiente de teste que já possua PVCs do Langfuse inicializados anteriormente com outro projeto ou outro par de API keys, `validate_langfuse_integration.sh` pode retornar erro de autenticação mesmo que `/api/public/health` esteja `200`. + +Primeiro confirme sem revelar a secret key: + +```bash +kubectl -n langfuse get secret langfuse-runtime \ + -o jsonpath='{.data.LANGFUSE_INIT_PROJECT_PUBLIC_KEY}' | base64 -d; echo + +kubectl -n agent-load-test get secret langfuse-client \ + -o jsonpath='{.data.LANGFUSE_PUBLIC_KEY}' | base64 -d; echo +``` + +As public keys precisam ser iguais. Depois execute: + +```bash +./deploy/oke/load-test/scripts/validate_langfuse_integration.sh +``` + +Se as keys estiverem sincronizadas no Kubernetes mas a API autenticada falhar, o banco persistente do Langfuse provavelmente já contém credenciais diferentes. Para um ambiente descartável de load test, a opção mais limpa é recriar a stack/PVCs do Langfuse e executar `deploy_langfuse.sh` novamente. Não apague PVCs de um Langfuse que contenha dados que precisem ser preservados. + +Depois de qualquer alteração de keys, execute novamente: + +```bash +./deploy/oke/load-test/scripts/deploy_backend.sh +``` + +O checksum de runtime força a criação de pods novos, garantindo que as novas credenciais entrem no environment do processo Python. + + +## Precedência das variáveis de ambiente + +`prepare_env.sh` usa o `.env` real do backend como fonte principal. A ordem é: + +```text +templates/agent_template_backend/.env (fonte principal) + ↓ +.env.loadtest.example (somente defaults ausentes) + ↓ +.env.loadtest (override explícito opcional) + ↓ +overrides obrigatórios OKE/container (wallet e Langfuse interno) + ↓ +.env.runtime +``` + +Portanto, `ENABLE_MCP_TOOLS`, `ENABLE_ANALYTICS`, endpoints, banco, modelos e demais configurações do agente são preservados por padrão. Para substituir deliberadamente alguma delas no teste, copie `.env.loadtest.override.example` para `.env.loadtest` e altere somente as chaves desejadas. + +Depois de executar: + +```bash +./deploy/oke/load-test/scripts/prepare_env.sh +``` + +o script informa quantas variáveis do `.env` do backend foram preservadas e lista apenas as que precisaram mudar por compatibilidade com OKE/container. diff --git a/deploy/oke/load-test/k8s/agent-backend.yaml b/deploy/oke/load-test/k8s/agent-backend.yaml new file mode 100644 index 0000000..31bce29 --- /dev/null +++ b/deploy/oke/load-test/k8s/agent-backend.yaml @@ -0,0 +1,190 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: agent-load-test +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-template-backend + namespace: agent-load-test +spec: + replicas: 4 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 2 + selector: + matchLabels: + app: agent-template-backend + template: + metadata: + labels: + app: agent-template-backend + annotations: + loadtest.runtime-checksum: "RUNTIME_CHECKSUM_PLACEHOLDER" + spec: + terminationGracePeriodSeconds: 60 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app: agent-template-backend + containers: + - name: agent-template-backend + image: IMAGE_PLACEHOLDER + imagePullPolicy: Always + ports: + - containerPort: 8000 + envFrom: + - secretRef: + name: agent-backend-runtime + env: + - name: LANGFUSE_PUBLIC_KEY + valueFrom: + secretKeyRef: + name: langfuse-client + key: LANGFUSE_PUBLIC_KEY + - name: LANGFUSE_SECRET_KEY + valueFrom: + secretKeyRef: + name: langfuse-client + key: LANGFUSE_SECRET_KEY + - name: LANGFUSE_HOST + valueFrom: + secretKeyRef: + name: langfuse-client + key: LANGFUSE_HOST + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 45 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + startupProbe: + httpGet: + path: /health + port: 8000 + periodSeconds: 5 + failureThreshold: 36 + resources: + requests: + cpu: "500m" + memory: "768Mi" + limits: + cpu: "2000m" + memory: "2Gi" + volumeMounts: + - name: wallet + mountPath: /app/wallet + readOnly: true + volumes: + - name: wallet + secret: + secretName: agent-backend-wallet +--- +apiVersion: v1 +kind: Service +metadata: + name: agent-template-backend + namespace: agent-load-test +spec: + type: ClusterIP + sessionAffinity: None + selector: + app: agent-template-backend + ports: + - name: http + port: 8000 + targetPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: agent-template-backend-lb + namespace: agent-load-test +#spec: +# type: LoadBalancer +# sessionAffinity: None +# selector: +# app: agent-template-backend +# ports: +# - name: http +# port: 8000 +# targetPort: 8000 +spec: + type: NodePort + selector: + app: agent-template-backend + ports: + - name: http + port: 8000 + targetPort: 8000 + nodePort: NODE_PORT_PLACEHOLDER +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: agent-template-backend + namespace: agent-load-test +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: agent-template-backend + minReplicas: 4 + maxReplicas: 30 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 60 + - type: Pods + value: 8 + periodSeconds: 60 + selectPolicy: Max + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 25 + periodSeconds: 60 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 65 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 75 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: agent-template-backend + namespace: agent-load-test +spec: + minAvailable: 3 + selector: + matchLabels: + app: agent-template-backend diff --git a/deploy/oke/load-test/k8s/k6-job.yaml b/deploy/oke/load-test/k8s/k6-job.yaml new file mode 100644 index 0000000..85fb36f --- /dev/null +++ b/deploy/oke/load-test/k8s/k6-job.yaml @@ -0,0 +1,28 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: agent-load-generator + namespace: agent-load-test +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app: agent-load-generator + spec: + restartPolicy: Never + containers: + - name: k6 + image: grafana/k6:latest + args: ["run", "/scripts/loadtest.js"] + envFrom: + - secretRef: + name: agent-backend-runtime + volumeMounts: + - name: script + mountPath: /scripts + volumes: + - name: script + configMap: + name: k6-load-script diff --git a/deploy/oke/load-test/langfuse/langfuse-k8s.yaml b/deploy/oke/load-test/langfuse/langfuse-k8s.yaml new file mode 100644 index 0000000..1d2e29a --- /dev/null +++ b/deploy/oke/load-test/langfuse/langfuse-k8s.yaml @@ -0,0 +1,633 @@ +# OKE load-test Langfuse manifest v11 - web HOSTNAME bind fix +apiVersion: v1 +kind: Namespace +metadata: + name: langfuse +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongo-data + namespace: langfuse +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-data + namespace: langfuse +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clickhouse-data + namespace: langfuse +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: clickhouse-logs + namespace: langfuse +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-data + namespace: langfuse +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: langfuse-common + namespace: langfuse +data: + NEXTAUTH_URL: http://localhost:3005 + POSTGRES_USER: postgres + POSTGRES_DB: postgres + TELEMETRY_ENABLED: 'false' + LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES: 'true' + CLICKHOUSE_MIGRATION_URL: clickhouse://clickhouse.langfuse.svc.cluster.local:9000 + CLICKHOUSE_URL: http://clickhouse.langfuse.svc.cluster.local:8123 + CLICKHOUSE_USER: clickhouse + CLICKHOUSE_CLUSTER_ENABLED: 'false' + REDIS_HOST: redis.langfuse.svc.cluster.local + REDIS_PORT: '6379' + REDIS_TLS_ENABLED: 'false' + LANGFUSE_USE_AZURE_BLOB: 'false' + LANGFUSE_S3_EVENT_UPLOAD_BUCKET: langfuse + LANGFUSE_S3_EVENT_UPLOAD_REGION: auto + LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID: minio + LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT: http://minio.langfuse.svc.cluster.local:9000 + LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE: 'true' + LANGFUSE_S3_EVENT_UPLOAD_PREFIX: events/ + LANGFUSE_S3_MEDIA_UPLOAD_BUCKET: langfuse + LANGFUSE_S3_MEDIA_UPLOAD_REGION: auto + LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID: minio + LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT: http://minio.langfuse.svc.cluster.local:9000 + LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE: 'true' + LANGFUSE_S3_MEDIA_UPLOAD_PREFIX: media/ + LANGFUSE_S3_BATCH_EXPORT_ENABLED: 'false' + LANGFUSE_S3_BATCH_EXPORT_BUCKET: langfuse + LANGFUSE_S3_BATCH_EXPORT_PREFIX: exports/ + LANGFUSE_S3_BATCH_EXPORT_REGION: auto + LANGFUSE_S3_BATCH_EXPORT_ENDPOINT: http://minio.langfuse.svc.cluster.local:9000 + LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT: http://localhost:9090 + LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID: minio + LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE: 'true' +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongo + namespace: langfuse +spec: + replicas: 1 + selector: + matchLabels: + app: mongo + template: + metadata: + labels: + app: mongo + spec: + containers: + - name: mongo + image: docker.io/library/mongo:8.0 + imagePullPolicy: IfNotPresent + env: + - name: MONGO_INITDB_ROOT_USERNAME + value: mongo + - name: MONGO_INITDB_ROOT_PASSWORD + value: mongopassword + - name: MONGO_INITDB_DATABASE + value: agent_memory + ports: + - name: mongo + containerPort: 27017 + volumeMounts: + - name: data + mountPath: /data/db + startupProbe: + exec: + command: + - sh + - -c + - 'mongosh --quiet --host 127.0.0.1 --port 27017 -u mongo -p mongopassword + --authenticationDatabase admin --eval ''db.adminCommand({ ping: 1 }).ok'' + | grep 1' + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + exec: + command: + - sh + - -c + - 'mongosh --quiet --host 127.0.0.1 --port 27017 -u mongo -p mongopassword + --authenticationDatabase admin --eval ''db.adminCommand({ ping: 1 }).ok'' + | grep 1' + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 1Gi + volumes: + - name: data + persistentVolumeClaim: + claimName: mongo-data +--- +apiVersion: v1 +kind: Service +metadata: + name: mongo + namespace: langfuse +spec: + selector: + app: mongo + ports: + - name: mongo + port: 27017 + targetPort: 27017 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: langfuse +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: docker.io/library/postgres:17 + imagePullPolicy: IfNotPresent + env: + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_DB + value: postgres + - name: TZ + value: UTC + - name: PGTZ + value: UTC + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: langfuse-runtime + key: LANGFUSE_POSTGRES_PASSWORD + ports: + - name: postgres + containerPort: 5432 + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + startupProbe: + exec: + command: + - sh + - -c + - pg_isready -U postgres + periodSeconds: 3 + failureThreshold: 40 + readinessProbe: + exec: + command: + - sh + - -c + - pg_isready -U postgres + periodSeconds: 5 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 1Gi + volumes: + - name: data + persistentVolumeClaim: + claimName: postgres-data +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: langfuse +spec: + selector: + app: postgres + ports: + - name: postgres + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: langfuse +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: docker.io/library/redis:7 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + args: + - exec redis-server --requirepass "$LANGFUSE_REDIS_PASSWORD" --maxmemory-policy + noeviction + env: + - name: LANGFUSE_REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: langfuse-runtime + key: LANGFUSE_REDIS_PASSWORD + ports: + - name: redis + containerPort: 6379 + startupProbe: + exec: + command: + - sh + - -c + - redis-cli -a "$LANGFUSE_REDIS_PASSWORD" ping | grep PONG + periodSeconds: 3 + failureThreshold: 30 + readinessProbe: + exec: + command: + - sh + - -c + - redis-cli -a "$LANGFUSE_REDIS_PASSWORD" ping | grep PONG + periodSeconds: 5 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: langfuse +spec: + selector: + app: redis + ports: + - name: redis + port: 6379 + targetPort: 6379 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clickhouse + namespace: langfuse +spec: + replicas: 1 + selector: + matchLabels: + app: clickhouse + template: + metadata: + labels: + app: clickhouse + spec: + securityContext: + fsGroup: 101 + fsGroupChangePolicy: OnRootMismatch + containers: + - name: clickhouse + image: docker.io/clickhouse/clickhouse-server:latest + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 101 + runAsGroup: 101 + env: + - name: CLICKHOUSE_DB + value: default + - name: CLICKHOUSE_USER + value: clickhouse + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: langfuse-runtime + key: LANGFUSE_CLICKHOUSE_PASSWORD + ports: + - name: http + containerPort: 8123 + - name: native + containerPort: 9000 + volumeMounts: + - name: data + mountPath: /var/lib/clickhouse + - name: logs + mountPath: /var/log/clickhouse-server + startupProbe: + httpGet: + path: /ping + port: 8123 + periodSeconds: 5 + failureThreshold: 40 + readinessProbe: + httpGet: + path: /ping + port: 8123 + periodSeconds: 5 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: '2' + memory: 2Gi + volumes: + - name: data + persistentVolumeClaim: + claimName: clickhouse-data + - name: logs + persistentVolumeClaim: + claimName: clickhouse-logs +--- +apiVersion: v1 +kind: Service +metadata: + name: clickhouse + namespace: langfuse +spec: + selector: + app: clickhouse + ports: + - name: http + port: 8123 + targetPort: 8123 + - name: native + port: 9000 + targetPort: 9000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: langfuse +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: docker.io/minio/minio:latest + imagePullPolicy: IfNotPresent + command: + - sh + - -c + args: + - mkdir -p /data/langfuse && exec minio server --address ":9000" --console-address + ":9001" /data + env: + - name: MINIO_ROOT_USER + value: minio + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: langfuse-runtime + key: LANGFUSE_MINIO_PASSWORD + ports: + - name: api + containerPort: 9000 + - name: console + containerPort: 9001 + volumeMounts: + - name: data + mountPath: /data + startupProbe: + httpGet: + path: /minio/health/live + port: 9000 + periodSeconds: 3 + failureThreshold: 40 + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + periodSeconds: 5 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: '1' + memory: 1Gi + volumes: + - name: data + persistentVolumeClaim: + claimName: minio-data +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: langfuse +spec: + selector: + app: minio + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: langfuse-worker + namespace: langfuse +spec: + replicas: 1 + selector: + matchLabels: + app: langfuse-worker + template: + metadata: + labels: + app: langfuse-worker + spec: + initContainers: + - name: wait-dependencies + image: docker.io/library/busybox:1.36 + command: + - sh + - -c + args: + - until nc -z postgres.langfuse.svc.cluster.local 5432 && nc -z redis.langfuse.svc.cluster.local + 6379 && nc -z clickhouse.langfuse.svc.cluster.local 8123 && nc -z minio.langfuse.svc.cluster.local + 9000; do echo 'waiting for Langfuse dependencies via FQDN'; sleep 3; done + containers: + - name: worker + image: docker.io/langfuse/langfuse-worker:3 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + name: langfuse-common + - secretRef: + name: langfuse-runtime + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: '2' + memory: 4Gi + env: + - name: NODE_OPTIONS + value: --max-old-space-size=3072 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: langfuse-web + namespace: langfuse +spec: + replicas: 1 + selector: + matchLabels: + app: langfuse-web + template: + metadata: + labels: + app: langfuse-web + spec: + initContainers: + - name: wait-dependencies + image: docker.io/library/busybox:1.36 + command: + - sh + - -c + args: + - until nc -z postgres.langfuse.svc.cluster.local 5432 && nc -z redis.langfuse.svc.cluster.local + 6379 && nc -z clickhouse.langfuse.svc.cluster.local 8123 && nc -z minio.langfuse.svc.cluster.local + 9000; do echo 'waiting for Langfuse dependencies via FQDN'; sleep 3; done + containers: + - name: web + image: docker.io/langfuse/langfuse:3 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 3000 + envFrom: + - configMapRef: + name: langfuse-common + - secretRef: + name: langfuse-runtime + startupProbe: + httpGet: + path: /api/public/health + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /api/public/health + port: 3000 + periodSeconds: 10 + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: '2' + memory: 4Gi + env: + - name: HOSTNAME + value: 0.0.0.0 + - name: NODE_OPTIONS + value: --max-old-space-size=3072 +--- +apiVersion: v1 +kind: Service +metadata: + name: langfuse-web + namespace: langfuse +spec: + selector: + app: langfuse-web + ports: + - name: http + port: 3000 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: langfuse-ui + namespace: langfuse +spec: + type: ClusterIP + selector: + app: langfuse-web + ports: + - name: http + port: 3005 + targetPort: 3000 diff --git a/deploy/oke/load-test/loadgen/loadtest.js b/deploy/oke/load-test/loadgen/loadtest.js new file mode 100644 index 0000000..ab214b2 --- /dev/null +++ b/deploy/oke/load-test/loadgen/loadtest.js @@ -0,0 +1,75 @@ +import http from 'k6/http'; +import { check, sleep } from 'k6'; +import exec from 'k6/execution'; +import { Trend, Counter } from 'k6/metrics'; + +const target = __ENV.LOADTEST_TARGET || 'http://agent-template-backend.agent-load-test.svc.cluster.local:8000'; +const agentId = __ENV.LOADTEST_AGENT_ID || 'telecom_contas'; +const tenantId = __ENV.LOADTEST_TENANT_ID || 'loadtest'; +const scenario = __ENV.LOADTEST_SCENARIO || 'unique_sessions'; +const message = __ENV.LOADTEST_MESSAGE || 'Explique em uma frase o que é uma fatura de telecom.'; +const rps = Number(__ENV.LOADTEST_RPS || 100); +const duration = __ENV.LOADTEST_DURATION || '10m'; +const maxVUs = Number(__ENV.LOADTEST_MAX_VUS || 1000); + +const backendLatency = new Trend('agent_backend_latency', true); +const errors = new Counter('agent_backend_errors'); + +export const options = { + discardResponseBodies: false, + scenarios: { + requests: { + executor: 'constant-arrival-rate', + rate: rps, + timeUnit: '1s', + duration, + preAllocatedVUs: Math.min(maxVUs, Math.max(50, rps * 2)), + maxVUs, + }, + }, + thresholds: { + http_req_failed: ['rate<0.05'], + checks: ['rate>0.95'], + }, +}; + +function sessionId() { + if (scenario === 'shared_sessions') { + return `shared-${exec.vu.idInTest % 100}`; + } + return `load-${exec.scenario.iterationInTest}-${exec.vu.idInTest}`; +} + +export default function () { + const sid = sessionId(); + const mid = `msg-${exec.scenario.iterationInTest}-${Date.now()}`; + const payload = JSON.stringify({ + channel: 'web', + agent_id: agentId, + tenant_id: tenantId, + payload: { + text: message, + session_id: sid, + user_id: `user-${exec.vu.idInTest}`, + customer_id: `cust-${exec.vu.idInTest}`, + message_id: mid, + metadata: { load_test: true, scenario }, + }, + }); + const started = Date.now(); + const res = http.post(`${target}/gateway/message`, payload, { + headers: { + 'Content-Type': 'application/json', + 'X-Request-ID': mid, + 'X-Load-Test': 'true', + }, + timeout: __ENV.LOADTEST_HTTP_TIMEOUT || '180s', + }); + backendLatency.add(Date.now() - started); + const ok = check(res, { + 'status is 200': (r) => r.status === 200, + 'response has body': (r) => !!r.body && r.body.length > 0, + }); + if (!ok) errors.add(1); + sleep(Number(__ENV.LOADTEST_THINK_TIME || 0)); +} diff --git a/deploy/oke/load-test/results/.gitkeep b/deploy/oke/load-test/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/deploy/oke/load-test/scripts/00_prepare.sh b/deploy/oke/load-test/scripts/00_prepare.sh new file mode 100644 index 0000000..9510532 --- /dev/null +++ b/deploy/oke/load-test/scripts/00_prepare.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +"$DIR/prepare_env.sh" +echo +cat <<'TXT' +Next: +1) edit deploy/oke/load-test/.env.runtime +2) copy the real Autonomous wallet into templates/agent_template_backend/wallet/ +3) run configure_kubeconfig.sh +TXT diff --git a/deploy/oke/load-test/scripts/01_deploy_all.sh b/deploy/oke/load-test/scripts/01_deploy_all.sh new file mode 100644 index 0000000..6f6d72f --- /dev/null +++ b/deploy/oke/load-test/scripts/01_deploy_all.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +"$DIR/deploy_langfuse.sh" +"$DIR/build_push_backend.sh" +"$DIR/deploy_backend.sh" +"$DIR/configure_existing_lb.sh" +"$DIR/validate_dependencies.sh" +"$DIR/smoke_test.sh" external diff --git a/deploy/oke/load-test/scripts/__pycache__/prepare_env.cpython-313.pyc b/deploy/oke/load-test/scripts/__pycache__/prepare_env.cpython-313.pyc new file mode 100644 index 0000000..c77aeb7 Binary files /dev/null and b/deploy/oke/load-test/scripts/__pycache__/prepare_env.cpython-313.pyc differ diff --git a/deploy/oke/load-test/scripts/build_push_backend.sh b/deploy/oke/load-test/scripts/build_push_backend.sh new file mode 100644 index 0000000..360865c --- /dev/null +++ b/deploy/oke/load-test/scripts/build_push_backend.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +ENV="$ROOT/deploy/oke/load-test/.env.runtime" +set -a; source "$ENV"; set +a +IMAGE="${OCI_REGION_KEY}.ocir.io/${OCI_TENANCY_NAMESPACE}/${OCIR_REPOSITORY_PREFIX}/agent-template-backend:${IMAGE_TAG}" +echo "Building $IMAGE" +docker build -f "$ROOT/deploy/oke/dockerfiles/Dockerfile.agent-template-backend" -t "$IMAGE" "$ROOT" +docker push "$IMAGE" +echo "$IMAGE" > "$ROOT/deploy/oke/load-test/.backend-image" diff --git a/deploy/oke/load-test/scripts/check_load_balancing.sh b/deploy/oke/load-test/scripts/check_load_balancing.sh new file mode 100644 index 0000000..007a53b --- /dev/null +++ b/deploy/oke/load-test/scripts/check_load_balancing.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +set -a; source "$ROOT/deploy/oke/load-test/.env.runtime"; set +a +NS="${K8S_NAMESPACE:-agent-load-test}" +IP="$(kubectl -n "$NS" get svc agent-template-backend-lb -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" +[[ -n "$IP" ]] || { echo "LoadBalancer IP not assigned"; exit 3; } +URL="http://${IP}:8000/health" +echo "Sampling 30 requests from $URL" +for i in $(seq 1 30); do + curl -fsS -D - -o /dev/null "$URL" 2>/dev/null | awk -F': ' 'tolower($1)=="x-agent-pod" {gsub("\r", "", $2); print $2}' +done | sort | uniq -c | sort -nr diff --git a/deploy/oke/load-test/scripts/collect_results.sh b/deploy/oke/load-test/scripts/collect_results.sh new file mode 100644 index 0000000..d688461 --- /dev/null +++ b/deploy/oke/load-test/scripts/collect_results.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +set -a; source "$LT/.env.runtime"; set +a +NS="${K8S_NAMESPACE:-agent-load-test}" +STAMP="$(date +%Y%m%d-%H%M%S)" +OUT="$LT/results/$STAMP" +mkdir -p "$OUT" +kubectl -n "$NS" get pods -o wide > "$OUT/pods.txt" +kubectl -n "$NS" get hpa -o yaml > "$OUT/hpa.yaml" +kubectl -n "$NS" get deploy agent-template-backend -o yaml > "$OUT/deployment.yaml" +kubectl -n "$NS" get svc -o wide > "$OUT/services.txt" +kubectl -n "$NS" top pods > "$OUT/top-pods.txt" 2>&1 || true +kubectl top nodes > "$OUT/top-nodes.txt" 2>&1 || true +kubectl -n "$NS" logs job/agent-load-generator > "$OUT/k6.log" 2>&1 || true +kubectl -n "$NS" logs -l app=agent-template-backend --prefix --tail=5000 > "$OUT/backend.log" 2>&1 || true +kubectl -n "$NS" get events --sort-by='.lastTimestamp' > "$OUT/events.txt" 2>&1 || true +echo "$OUT" diff --git a/deploy/oke/load-test/scripts/configure_existing_lb.sh b/deploy/oke/load-test/scripts/configure_existing_lb.sh new file mode 100644 index 0000000..e5d3ee4 --- /dev/null +++ b/deploy/oke/load-test/scripts/configure_existing_lb.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +ENV="$LT/.env.runtime" +[[ -f "$ENV" ]] && source "$ENV" + +NS="${K8S_NAMESPACE:-agent-load-test}" +SERVICE_NAME="${BACKEND_SERVICE_NAME:-agent-template-backend-lb}" +LB_OCID="${EXISTING_LB_OCID:?EXISTING_LB_OCID must be set}" +BACKEND_SET="${BACKEND_SET_NAME:-agent-framework-loadtest}" +LISTENER_NAME="${BACKEND_LISTENER_NAME:-agent-framework-loadtest}" +LISTENER_PORT="${BACKEND_LISTENER_PORT:-8000}" +LISTENER_PROTOCOL="${BACKEND_LISTENER_PROTOCOL:-HTTP}" +BACKEND_POLICY="${BACKEND_POLICY:-ROUND_ROBIN}" +HEALTH_PATH="${BACKEND_HEALTH_PATH:-/health}" +OCI_PROFILE="${OCI_CLI_PROFILE:-}" + +log(){ printf '[existing-lb] %s\n' "$*"; } +die(){ printf '[existing-lb] ERROR: %s\n' "$*" >&2; exit 1; } + +OCI_ARGS=() +[[ -n "$OCI_PROFILE" ]] && OCI_ARGS+=(--profile "$OCI_PROFILE") +oci_lb(){ oci lb "$@" "${OCI_ARGS[@]}"; } + +command -v oci >/dev/null || die "oci CLI not found" +command -v kubectl >/dev/null || die "kubectl not found" + +log "Validating Load Balancer..." +oci_lb load-balancer get --load-balancer-id "$LB_OCID" >/dev/null + +NODE_PORT="$(kubectl -n "$NS" get svc "$SERVICE_NAME" -o jsonpath='{.spec.ports[0].nodePort}')" +[[ -n "$NODE_PORT" && "$NODE_PORT" != "0" ]] || die "Service/$SERVICE_NAME has no NodePort" +log "NodePort=$NODE_PORT" + +if oci_lb backend-set get --load-balancer-id "$LB_OCID" --backend-set-name "$BACKEND_SET" >/dev/null 2>&1; then + log "Backend set '$BACKEND_SET' already exists" +else + log "Creating backend set '$BACKEND_SET'..." + oci_lb backend-set create \ + --load-balancer-id "$LB_OCID" \ + --name "$BACKEND_SET" \ + --policy "$BACKEND_POLICY" \ + --health-checker-protocol HTTP \ + --health-checker-port "$NODE_PORT" \ + --health-checker-url-path "$HEALTH_PATH" \ + --health-checker-return-code 200 \ + --health-checker-retries 3 \ + --health-checker-timeout-in-ms 3000 \ + --health-checker-interval-in-ms 10000 \ + --wait-for-state SUCCEEDED \ + --max-wait-seconds 1200 >/dev/null +fi + +for i in $(seq 1 20); do + oci_lb backend-set get --load-balancer-id "$LB_OCID" --backend-set-name "$BACKEND_SET" >/dev/null 2>&1 && break + [[ "$i" == "20" ]] && die "backend set still unavailable" + sleep 3 +done + +mapfile -t NODE_IPS < <( + kubectl get nodes -o jsonpath='{range .items[*]}{.status.addresses[?(@.type=="InternalIP")].address}{"\n"}{end}' | + sed '/^$/d' | sort -u +) + +EXISTING="$(oci_lb backend list \ + --load-balancer-id "$LB_OCID" \ + --backend-set-name "$BACKEND_SET" \ + --query 'data[].name' --raw-output 2>/dev/null || true)" + +for ip in "${NODE_IPS[@]}"; do + name="${ip}:${NODE_PORT}" + if grep -Fxq "$name" <<<"$EXISTING"; then + log "Backend $name already exists" + else + log "Creating backend $name..." + oci_lb backend create \ + --load-balancer-id "$LB_OCID" \ + --backend-set-name "$BACKEND_SET" \ + --ip-address "$ip" \ + --port "$NODE_PORT" \ + --wait-for-state SUCCEEDED \ + --max-wait-seconds 1200 >/dev/null + fi +done + +listener_found="$( + oci_lb load-balancer get \ + --load-balancer-id "$LB_OCID" \ + --query "data.listeners.\"${LISTENER_NAME}\".name" \ + --raw-output 2>/dev/null || true +)" + +if [[ "$listener_found" == "$LISTENER_NAME" ]]; then + log "Listener '$LISTENER_NAME' already exists" +else + log "Creating listener '$LISTENER_NAME'..." + oci_lb listener create \ + --load-balancer-id "$LB_OCID" \ + --name "$LISTENER_NAME" \ + --default-backend-set-name "$BACKEND_SET" \ + --port "$LISTENER_PORT" \ + --protocol "$LISTENER_PROTOCOL" \ + --wait-for-state SUCCEEDED \ + --max-wait-seconds 1200 >/dev/null +fi + +log "Backends:" +oci_lb backend list \ + --load-balancer-id "$LB_OCID" \ + --backend-set-name "$BACKEND_SET" \ + --query 'data[].{name:name,ip:"ip-address",port:port}' \ + --output table + +log "Backend set health:" +oci_lb backend-set-health get \ + --load-balancer-id "$LB_OCID" \ + --backend-set-name "$BACKEND_SET" \ + --output table || true + +log "Done" diff --git a/deploy/oke/load-test/scripts/configure_kubeconfig.sh b/deploy/oke/load-test/scripts/configure_kubeconfig.sh new file mode 100644 index 0000000..0dbceac --- /dev/null +++ b/deploy/oke/load-test/scripts/configure_kubeconfig.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +ENV="$ROOT/deploy/oke/load-test/.env.runtime" +[[ -f "$ENV" ]] || { echo "Run prepare_env.sh first"; exit 1; } +set -a; source "$ENV"; set +a +oci ce cluster create-kubeconfig \ + --cluster-id "$OKE_CLUSTER_OCID" \ + --file "$HOME/.kube/config" \ + --region "$OCI_REGION" \ + --token-version 2.0.0 \ + --kube-endpoint PUBLIC_ENDPOINT \ + --profile "${OCI_CLI_PROFILE:-DEFAULT}" +kubectl cluster-info +kubectl get nodes -o wide diff --git a/deploy/oke/load-test/scripts/create_runtime_secret.sh b/deploy/oke/load-test/scripts/create_runtime_secret.sh new file mode 100644 index 0000000..5d924c7 --- /dev/null +++ b/deploy/oke/load-test/scripts/create_runtime_secret.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +ENV="$ROOT/deploy/oke/load-test/.env.runtime" +[[ -f "$ENV" ]] || { echo "Run prepare_env.sh first"; exit 1; } +set -a; source "$ENV"; set +a +NS="${K8S_NAMESPACE:-agent-load-test}" +kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - +kubectl -n "$NS" create secret generic agent-backend-runtime --from-env-file="$ENV" --dry-run=client -o yaml | kubectl apply -f - +echo "Runtime env loaded into secret agent-backend-runtime." diff --git a/deploy/oke/load-test/scripts/create_wallet_secret.sh b/deploy/oke/load-test/scripts/create_wallet_secret.sh new file mode 100644 index 0000000..20f484c --- /dev/null +++ b/deploy/oke/load-test/scripts/create_wallet_secret.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +ENV="$ROOT/deploy/oke/load-test/.env.runtime" +WALLET="$ROOT/templates/agent_template_backend/wallet" +set -a; source "$ENV"; set +a +NS="${K8S_NAMESPACE:-agent-load-test}" +mapfile -t wallet_files < <(find "$WALLET" -maxdepth 1 -type f ! -name 'README_PLACE_WALLET_HERE.txt' ! -name '.gitkeep') +if (( ${#wallet_files[@]} == 0 )); then + echo "No real wallet files found in $WALLET" + echo "Copy the Autonomous wallet files there, then rerun." + exit 2 +fi +kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - +args=() +for f in "${wallet_files[@]}"; do args+=("--from-file=$(basename "$f")=$f"); done +kubectl -n "$NS" create secret generic agent-backend-wallet "${args[@]}" --dry-run=client -o yaml | kubectl apply -f - +echo "Wallet secret agent-backend-wallet created/updated in namespace $NS" diff --git a/deploy/oke/load-test/scripts/deploy_backend.sh b/deploy/oke/load-test/scripts/deploy_backend.sh new file mode 100644 index 0000000..02aa00a --- /dev/null +++ b/deploy/oke/load-test/scripts/deploy_backend.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +ENV="$LT/.env.runtime" + +[[ -f "$ENV" ]] || { echo "ERROR: runtime env not found: $ENV" >&2; exit 1; } + +# Keep variables local to this shell; do not export everything to kubectl/OCI CLI. +source "$ENV" + +log() { printf '[deploy-backend] %s\n' "$*"; } +die() { printf '[deploy-backend] ERROR: %s\n' "$*" >&2; exit 1; } + +command -v kubectl >/dev/null 2>&1 || die "kubectl not found in PATH" + +NS="${K8S_NAMESPACE:-agent-load-test}" +OCIR_SECRET_NAME="${OCIR_PULL_SECRET_NAME:-ocir-secret}" + +IMAGE="$(cat "$LT/.backend-image" 2>/dev/null || true)" +[[ -n "$IMAGE" ]] || IMAGE="${OCI_REGION_KEY}.ocir.io/${OCI_TENANCY_NAMESPACE}/${OCIR_REPOSITORY_PREFIX}/agent-template-backend:${IMAGE_TAG}" + +log "Namespace: $NS" +log "Backend image: $IMAGE" + +log "Checking Kubernetes API authentication..." +kubectl get --raw='/readyz' >/dev/null 2>&1 || die "Kubernetes API authentication failed" +kubectl get --raw='/openapi/v2' >/dev/null 2>&1 || die "Kubernetes OpenAPI access failed" + +log "Ensuring namespace exists..." +kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +: "${OCI_REGION_KEY:?OCI_REGION_KEY must be set in $ENV}" +: "${OCIR_USERNAME:?OCIR_USERNAME must be set in $ENV}" +: "${OCIR_AUTH_TOKEN:?OCIR_AUTH_TOKEN must be set in $ENV}" + +OCIR_EMAIL="${OCIR_EMAIL:-unused@example.invalid}" +OCIR_SERVER="${OCI_REGION_KEY}.ocir.io" + +log "Creating/updating OCIR pull secret '$OCIR_SECRET_NAME'..." +kubectl create secret docker-registry "$OCIR_SECRET_NAME" -n "$NS" --docker-server="$OCIR_SERVER" --docker-username="$OCIR_USERNAME" --docker-password="$OCIR_AUTH_TOKEN" --docker-email="$OCIR_EMAIL" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +log "Associating OCIR pull secret with ServiceAccount/default..." +kubectl patch serviceaccount default -n "$NS" --type=merge -p "{\"imagePullSecrets\":[{\"name\":\"${OCIR_SECRET_NAME}\"}]}" >/dev/null + +pull_secret="$(kubectl get serviceaccount default -n "$NS" -o jsonpath='{.imagePullSecrets[*].name}')" +[[ " $pull_secret " == *" $OCIR_SECRET_NAME "* ]] || die "ServiceAccount/default does not reference $OCIR_SECRET_NAME" + +log "Creating/updating backend runtime secret..." +"$LT/scripts/create_runtime_secret.sh" + +if [[ "${ENABLE_LANGFUSE:-true}" == "true" ]]; then + log "Synchronizing Langfuse client credentials..." + "$LT/scripts/sync_langfuse_client_secret.sh" +fi + +log "Creating/updating backend wallet secret..." +"$LT/scripts/create_wallet_secret.sh" + +RUNTIME_CHECKSUM="$(cat "$LT/.env.runtime" "$LT/.env.langfuse" 2>/dev/null | sha256sum | awk '{print $1}')" +NODE_PORT="${BACKEND_NODE_PORT:-32116}" + +log "Applying backend Kubernetes manifest..." +sed -e "s#namespace: agent-load-test#namespace: $NS#g" -e "s#name: agent-load-test#name: $NS#g" -e "s#IMAGE_PLACEHOLDER#$IMAGE#g" -e "s#replicas: 4#replicas: ${LOADTEST_MIN_REPLICAS:-4}#" -e "s#minReplicas: 4#minReplicas: ${LOADTEST_MIN_REPLICAS:-4}#" -e "s#maxReplicas: 30#maxReplicas: ${LOADTEST_MAX_REPLICAS:-30}#" -e "s#cpu: \"500m\"#cpu: \"${LOADTEST_CPU_REQUEST:-500m}\"#" -e "s#cpu: \"2000m\"#cpu: \"${LOADTEST_CPU_LIMIT:-2000m}\"#" -e "s#memory: \"768Mi\"#memory: \"${LOADTEST_MEMORY_REQUEST:-768Mi}\"#" -e "s#memory: \"2Gi\"#memory: \"${LOADTEST_MEMORY_LIMIT:-2Gi}\"#" -e "s#RUNTIME_CHECKSUM_PLACEHOLDER#$RUNTIME_CHECKSUM#g" -e "s#NODE_PORT_PLACEHOLDER#$NODE_PORT#g" "$LT/k8s/agent-backend.yaml" | kubectl apply -f - + +log "Waiting for backend rollout..." +if ! kubectl -n "$NS" rollout status deployment/agent-template-backend --timeout=10m; then + echo "================ BACKEND ROLLOUT FAILED ================" >&2 + kubectl -n "$NS" get pods -o wide >&2 || true + bad_pod="$(kubectl -n "$NS" get pods -l app=agent-template-backend --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[-1].metadata.name}' 2>/dev/null || true)" + if [[ -n "$bad_pod" ]]; then + kubectl -n "$NS" describe pod "$bad_pod" >&2 || true + kubectl -n "$NS" logs "$bad_pod" -c agent-template-backend --tail=200 >&2 || true + fi + exit 1 +fi + +log "Final backend pods:" +kubectl -n "$NS" get pods -o wide + +log "Backend NodePort service (fronted by existing OCI LB):" +kubectl -n "$NS" get svc agent-template-backend-lb + +log "Backend HPA:" +kubectl -n "$NS" get hpa + +log "Default ServiceAccount imagePullSecrets:" +kubectl -n "$NS" get serviceaccount default -o jsonpath='{.imagePullSecrets[*].name}{"\n"}' + +log "Backend deployment completed successfully." diff --git a/deploy/oke/load-test/scripts/deploy_langfuse.sh b/deploy/oke/load-test/scripts/deploy_langfuse.sh new file mode 100644 index 0000000..4f0b644 --- /dev/null +++ b/deploy/oke/load-test/scripts/deploy_langfuse.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# deploy_langfuse_v11.sh +# OKE-safe Langfuse deployment with DNS preflight. +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +MANIFEST="$LT/langfuse/langfuse-k8s.yaml" +LF_ENV="$LT/.env.langfuse" + +NS="${LANGFUSE_NAMESPACE:-langfuse}" +SECRET="langfuse-runtime" +DNS_TEST_NS="${DNS_TEST_NAMESPACE:-default}" + +PG_FQDN="postgres.${NS}.svc.cluster.local" +REDIS_FQDN="redis.${NS}.svc.cluster.local" +CH_FQDN="clickhouse.${NS}.svc.cluster.local" +MINIO_FQDN="minio.${NS}.svc.cluster.local" +MONGO_FQDN="mongo.${NS}.svc.cluster.local" + +log() { printf '[langfuse-deploy] %s\n' "$*"; } +die() { printf '[langfuse-deploy] ERROR: %s\n' "$*" >&2; exit 1; } + +cleanup_probe() { + local ns="${1:-$DNS_TEST_NS}" + local name="${2:-}" + [[ -n "$name" ]] && kubectl -n "$ns" delete pod "$name" --ignore-not-found --wait=false >/dev/null 2>&1 || true +} + +trap 'rc=$?; if (( rc != 0 )); then + echo >&2 + echo "[langfuse-deploy] Failure detected." >&2 + kubectl get nodes -o wide 2>/dev/null >&2 || true + kubectl -n kube-system get pods -l k8s-app=kube-dns -o wide 2>/dev/null >&2 || true + kubectl -n "$NS" get pods,svc,pvc 2>/dev/null >&2 || true +fi +exit $rc' EXIT + +command -v kubectl >/dev/null 2>&1 || die "kubectl not found in PATH" +[[ -f "$MANIFEST" ]] || die "manifest not found: $MANIFEST" + +log "Checking Kubernetes API..." +kubectl version --request-timeout=15s >/dev/null || die "cannot reach Kubernetes API" + +log "Checking worker nodes..." +not_ready="$(kubectl get nodes --no-headers 2>/dev/null | awk '$2 != "Ready" {print $1 ":" $2}')" +[[ -z "$not_ready" ]] || { printf '%s\n' "$not_ready" >&2; die "one or more OKE nodes are not Ready"; } + +log "Checking CoreDNS..." +kubectl -n kube-system get svc kube-dns >/dev/null 2>&1 || die "kube-dns Service not found" +dns_ready="$(kubectl -n kube-system get pods -l k8s-app=kube-dns --no-headers 2>/dev/null | awk '$2 ~ /^[0-9]+\/[0-9]+$/ {split($2,a,"/"); if(a[1]==a[2]) n++} END{print n+0}')" +(( dns_ready > 0 )) || die "no Ready CoreDNS pod found" + +DNS_IP="$(kubectl -n kube-system get svc kube-dns -o jsonpath='{.spec.clusterIP}')" +[[ -n "$DNS_IP" ]] || die "kube-dns ClusterIP is empty" +log "kube-dns ClusterIP: $DNS_IP" + +probe="oke-dns-preflight-$(date +%s)" +log "Running DNS preflight BEFORE Langfuse deployment..." +kubectl -n "$DNS_TEST_NS" run "$probe" --restart=Never --image=docker.io/library/busybox:1.36 --command -- sh -c "echo '--- /etc/resolv.conf ---'; + cat /etc/resolv.conf; + echo '--- default resolver ---'; + nslookup kubernetes.default.svc.cluster.local; + echo '--- direct kube-dns ---'; + nslookup kubernetes.default.svc.cluster.local ${DNS_IP}" >/dev/null + +if ! kubectl -n "$DNS_TEST_NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$probe" --timeout=90s >/dev/null 2>&1; then + echo >&2 + echo "================ DNS PREFLIGHT FAILED ================" >&2 + kubectl -n "$DNS_TEST_NS" logs "$probe" >&2 || true + kubectl -n "$DNS_TEST_NS" describe pod "$probe" >&2 || true + cleanup_probe "$DNS_TEST_NS" "$probe" + die "cluster DNS is unavailable; fix worker/CNI networking before deploying Langfuse" +fi +kubectl -n "$DNS_TEST_NS" logs "$probe" || true +cleanup_probe "$DNS_TEST_NS" "$probe" +log "Cluster DNS preflight PASSED." + +log "Generating Langfuse runtime environment..." +"$LT/scripts/prepare_env.sh" +[[ -s "$LF_ENV" ]] || die "Langfuse env was not generated: $LF_ENV" + +required_keys=( + LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY + LANGFUSE_INIT_PROJECT_PUBLIC_KEY LANGFUSE_INIT_PROJECT_SECRET_KEY + LANGFUSE_INIT_ORG_ID LANGFUSE_INIT_ORG_NAME + LANGFUSE_INIT_PROJECT_ID LANGFUSE_INIT_PROJECT_NAME + LANGFUSE_INIT_USER_EMAIL LANGFUSE_INIT_USER_NAME + LANGFUSE_INIT_USER_PASSWORD LANGFUSE_NEXTAUTH_SECRET + LANGFUSE_SALT LANGFUSE_ENCRYPTION_KEY + LANGFUSE_POSTGRES_PASSWORD LANGFUSE_REDIS_PASSWORD + LANGFUSE_CLICKHOUSE_PASSWORD LANGFUSE_MINIO_PASSWORD +) + +for key in "${required_keys[@]}"; do + grep -qE "^${key}=.+" "$LF_ENV" || die "required key '$key' missing/empty in $LF_ENV" +done + +kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +pg_password="$(grep '^LANGFUSE_POSTGRES_PASSWORD=' "$LF_ENV" | cut -d= -f2-)" +[[ -n "$pg_password" ]] || die "LANGFUSE_POSTGRES_PASSWORD is empty" +db_url="postgresql://postgres:${pg_password}@${PG_FQDN}:5432/postgres" + +tmp_env="$(mktemp)" +grep -vE '^(DATABASE_URL|DIRECT_URL)=' "$LF_ENV" > "$tmp_env" +printf 'DATABASE_URL=%s\n' "$db_url" >> "$tmp_env" +printf 'DIRECT_URL=%s\n' "$db_url" >> "$tmp_env" + +log "Creating/updating Secret/$SECRET..." +kubectl -n "$NS" create secret generic "$SECRET" --from-env-file="$tmp_env" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +rm -f "$tmp_env" + +log "Applying Langfuse manifest..." +kubectl apply -f "$MANIFEST" + +for dep in mongo postgres redis clickhouse minio langfuse-worker langfuse-web; do + kubectl -n "$NS" get deployment "$dep" >/dev/null 2>&1 || die "deployment/$dep is missing" +done + +pgdata="$(kubectl -n "$NS" get deployment postgres -o jsonpath='{.spec.template.spec.containers[?(@.name=="postgres")].env[?(@.name=="PGDATA")].value}')" +[[ "$pgdata" == "/var/lib/postgresql/data/pgdata" ]] || die "Postgres PGDATA is incorrect: '$pgdata'" + +log "Scaling web/worker to zero until infrastructure is healthy..." +kubectl -n "$NS" scale deployment/langfuse-web --replicas=0 >/dev/null +kubectl -n "$NS" scale deployment/langfuse-worker --replicas=0 >/dev/null + +for dep in mongo postgres redis clickhouse minio; do + log "Waiting for deployment/$dep ..." + kubectl -n "$NS" rollout status "deployment/$dep" --timeout=10m +done + +log "Checking service endpoints..." +for svc in mongo postgres redis clickhouse minio; do + ip="$(kubectl -n "$NS" get endpoints "$svc" -o jsonpath='{.subsets[0].addresses[0].ip}' 2>/dev/null || true)" + [[ -n "$ip" ]] || die "service/$svc has no ready endpoint" + log "$svc endpoint: $ip" +done + +probe="langfuse-infra-probe-$(date +%s)" +log "Running Langfuse infrastructure DNS/TCP probe..." +kubectl -n "$NS" run "$probe" --restart=Never --image=docker.io/library/busybox:1.36 --command -- sh -c "set -e; + nslookup ${PG_FQDN}; + nslookup ${REDIS_FQDN}; + nslookup ${CH_FQDN}; + nslookup ${MINIO_FQDN}; + nslookup ${MONGO_FQDN}; + nc -zvw5 ${PG_FQDN} 5432; + nc -zvw5 ${REDIS_FQDN} 6379; + nc -zvw5 ${CH_FQDN} 8123; + nc -zvw5 ${MINIO_FQDN} 9000; + nc -zvw5 ${MONGO_FQDN} 27017; + echo 'ALL LANGFUSE INFRA CHECKS PASSED'" >/dev/null + +if ! kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$probe" --timeout=120s >/dev/null 2>&1; then + kubectl -n "$NS" logs "$probe" >&2 || true + kubectl -n "$NS" describe pod "$probe" >&2 || true + cleanup_probe "$NS" "$probe" + die "Langfuse infrastructure probe failed; web/worker remain scaled to zero" +fi +kubectl -n "$NS" logs "$probe" || true +cleanup_probe "$NS" "$probe" + +log "Infrastructure healthy. Applying Langfuse web/worker memory sizing..." +kubectl -n "$NS" set resources deployment/langfuse-worker --requests=cpu=500m,memory=2Gi --limits=cpu=2,memory=4Gi >/dev/null +kubectl -n "$NS" set resources deployment/langfuse-web --requests=cpu=500m,memory=2Gi --limits=cpu=2,memory=4Gi >/dev/null +kubectl -n "$NS" set env deployment/langfuse-worker NODE_OPTIONS=--max-old-space-size=3072 >/dev/null +kubectl -n "$NS" set env deployment/langfuse-web NODE_OPTIONS=--max-old-space-size=3072 HOSTNAME=0.0.0.0 >/dev/null + +log "Scaling worker/web to 1..." +kubectl -n "$NS" scale deployment/langfuse-worker --replicas=1 >/dev/null +kubectl -n "$NS" scale deployment/langfuse-web --replicas=1 >/dev/null + +kubectl -n "$NS" rollout status deployment/langfuse-worker --timeout=15m +kubectl -n "$NS" rollout status deployment/langfuse-web --timeout=15m + +log "Verifying Secret/$SECRET before synchronization..." +kubectl -n "$NS" get secret "$SECRET" >/dev/null || die "Secret/$SECRET became unavailable in namespace $NS" + +APP_NS="$(sed -n -E 's/^K8S_NAMESPACE=(.*)$/\1/p' "$LT/.env.runtime" | tail -n 1 | tr -d '\r' | sed -e 's/^"//' -e 's/"$//' -e "s/^'//" -e "s/'$//")" +APP_NS="${APP_NS:-agent-load-test}" + +log "Synchronizing Langfuse project keys to backend namespace $APP_NS..." +LANGFUSE_NAMESPACE="$NS" K8S_NAMESPACE="$APP_NS" \ + "$LT/scripts/sync_langfuse_client_secret.sh" + +log "Validating Langfuse health and authenticated Public API..." +"$LT/scripts/validate_langfuse_integration.sh" + +log "Final state:" +kubectl -n "$NS" get pods,svc,pvc -o wide +printf '\nUI: kubectl -n %s port-forward --address 127.0.0.1 svc/langfuse-web 3005:3000\n' "$NS" +log "Langfuse deployment completed successfully." diff --git a/deploy/oke/load-test/scripts/fix_oke_worker_network_v1.sh b/deploy/oke/load-test/scripts/fix_oke_worker_network_v1.sh new file mode 100644 index 0000000..e139315 --- /dev/null +++ b/deploy/oke/load-test/scripts/fix_oke_worker_network_v1.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# fix_oke_worker_network_v1.sh +# Run ON EACH OKE WORKER NODE as root or with sudo. +set -Eeuo pipefail + +log() { printf '[oke-worker-network] %s\n' "$*"; } + +if [[ "$(id -u)" -ne 0 ]]; then + echo "ERROR: run this script with sudo/root." >&2 + exit 1 +fi + +log "Loading br_netfilter..." +modprobe br_netfilter + +log "Persisting br_netfilter..." +cat >/etc/modules-load.d/br_netfilter.conf <<'EOF' +br_netfilter +EOF + +log "Applying Kubernetes networking sysctls..." +cat >/etc/sysctl.d/99-kubernetes-network.conf <<'EOF' +net.bridge.bridge-nf-call-iptables=1 +net.bridge.bridge-nf-call-ip6tables=1 +net.ipv4.ip_forward=1 +EOF + +sysctl --system >/dev/null + +log "Validating kernel/module settings..." +lsmod | grep -q '^br_netfilter' || { echo "ERROR: br_netfilter is not loaded." >&2; exit 1; } +[[ "$(sysctl -n net.bridge.bridge-nf-call-iptables)" == "1" ]] || { echo "ERROR: bridge-nf-call-iptables != 1" >&2; exit 1; } +[[ "$(sysctl -n net.ipv4.ip_forward)" == "1" ]] || { echo "ERROR: ip_forward != 1" >&2; exit 1; } + +log "Restarting CRI-O and kubelet..." +systemctl restart crio +systemctl restart kubelet + +log "Worker network correction completed." diff --git a/deploy/oke/load-test/scripts/prepare_env.py b/deploy/oke/load-test/scripts/prepare_env.py new file mode 100644 index 0000000..fee5b98 --- /dev/null +++ b/deploy/oke/load-test/scripts/prepare_env.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import argparse +from pathlib import Path +import secrets +import shlex + + +def parse_env(path: Path) -> dict[str, str]: + out: dict[str, str] = {} + if not path.exists(): + return out + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + # Support the common `export KEY=value` form too. + if line.startswith("export "): + line = line[7:].lstrip() + k, v = line.split("=", 1) + k = k.strip() + if not k: + continue + out[k] = v.strip() + return out + + +def write_env(path: Path, values: dict[str, str], *, shell_safe: bool = False): + lines = ["# Generated by prepare_env.py. Do not commit."] + for k in sorted(values): + value = values[k] + if shell_safe: + value = shlex.quote(value) + lines.append(f"{k}={value}") + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def merge_missing(target: dict[str, str], defaults: dict[str, str]) -> None: + """Add only keys that are missing from target; never replace backend values.""" + for k, v in defaults.items(): + target.setdefault(k, v) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base", required=True, + help="Main application env. Existing values are preserved.") + p.add_argument("--overlay", required=True, + help="Load-test defaults. Only fills keys absent from --base.") + p.add_argument("--override", required=False, + help="Optional explicit load-test overrides. These replace base/default values.") + p.add_argument("--output", required=True) + p.add_argument("--langfuse-env", required=True) + args = p.parse_args() + + base_path = Path(args.base) + defaults_path = Path(args.overlay) + override_path = Path(args.override) if args.override else None + + base = parse_env(base_path) + defaults = parse_env(defaults_path) + explicit = parse_env(override_path) if override_path and override_path.exists() else {} + + # Backend .env is authoritative. The example file provides only missing load-test keys. + merged = dict(base) + merge_missing(merged, defaults) + + # .env.loadtest, when present, is the deliberate operator override layer. + merged.update(explicit) + + # Container/OKE invariants. These are location/routing details that cannot use + # workstation-local values from templates/agent_template_backend/.env. + merged["APP_ENV"] = "oke-load-test" + merged["ADB_WALLET_LOCATION"] = "/app/wallet" + merged["ENABLE_LANGFUSE"] = "true" + merged["LANGFUSE_HOST"] = "http://langfuse-web.langfuse.svc.cluster.local:3000" + + # OKE authentication should normally use workload identity. Allow an explicit + # .env.loadtest override if the operator intentionally selected another mode. + if "OCI_AUTH_MODE" not in explicit: + merged["OCI_AUTH_MODE"] = "oke_workload_identity" + + # Correct only the historical typo in Mongo-compatible ports; preserve host, + # credentials, query params and database values from the backend .env. + for key in ("MONGODB_URI", "PUBSUB_SEQUENCE_MONGODB_URI"): + if key in merged: + merged[key] = merged[key].replace(":37017", ":27017") + + # Langfuse bootstrap secrets are stable across reruns. + lf_path = Path(args.langfuse_env) + lf = parse_env(lf_path) + public_key = lf.get("LANGFUSE_PUBLIC_KEY") or "pk-lf-" + secrets.token_hex(16) + secret_key = lf.get("LANGFUSE_SECRET_KEY") or "sk-lf-" + secrets.token_hex(32) + admin_password = lf.get("LANGFUSE_INIT_USER_PASSWORD") or secrets.token_urlsafe(24) + + lf.update({ + "LANGFUSE_INIT_ORG_ID": lf.get("LANGFUSE_INIT_ORG_ID", "agent-framework-org"), + "LANGFUSE_INIT_ORG_NAME": lf.get("LANGFUSE_INIT_ORG_NAME", "Agent Framework OCI"), + "LANGFUSE_INIT_PROJECT_ID": lf.get("LANGFUSE_INIT_PROJECT_ID", "agent-framework-loadtest"), + "LANGFUSE_INIT_PROJECT_NAME": lf.get("LANGFUSE_INIT_PROJECT_NAME", "Agent Framework Load Test"), + "LANGFUSE_PUBLIC_KEY": public_key, + "LANGFUSE_SECRET_KEY": secret_key, + "LANGFUSE_INIT_USER_EMAIL": lf.get("LANGFUSE_INIT_USER_EMAIL", "admin@loadtest.local"), + "LANGFUSE_INIT_USER_NAME": lf.get("LANGFUSE_INIT_USER_NAME", "Load Test Admin"), + "LANGFUSE_INIT_USER_PASSWORD": admin_password, + "LANGFUSE_NEXTAUTH_SECRET": lf.get("LANGFUSE_NEXTAUTH_SECRET") or secrets.token_hex(32), + "LANGFUSE_SALT": lf.get("LANGFUSE_SALT") or secrets.token_hex(16), + "LANGFUSE_ENCRYPTION_KEY": lf.get("LANGFUSE_ENCRYPTION_KEY") or secrets.token_hex(32), + "LANGFUSE_POSTGRES_PASSWORD": lf.get("LANGFUSE_POSTGRES_PASSWORD") or secrets.token_urlsafe(24), + "LANGFUSE_REDIS_PASSWORD": lf.get("LANGFUSE_REDIS_PASSWORD") or secrets.token_urlsafe(24), + "LANGFUSE_CLICKHOUSE_PASSWORD": lf.get("LANGFUSE_CLICKHOUSE_PASSWORD") or secrets.token_urlsafe(24), + "LANGFUSE_MINIO_PASSWORD": lf.get("LANGFUSE_MINIO_PASSWORD") or secrets.token_urlsafe(24), + }) + lf.update({ + "DATABASE_URL": f"postgresql://postgres:{lf['LANGFUSE_POSTGRES_PASSWORD']}@postgres.langfuse.svc.cluster.local:5432/postgres", + "DIRECT_URL": f"postgresql://postgres:{lf['LANGFUSE_POSTGRES_PASSWORD']}@postgres.langfuse.svc.cluster.local:5432/postgres", + "POSTGRES_PASSWORD": lf["LANGFUSE_POSTGRES_PASSWORD"], + "SALT": lf["LANGFUSE_SALT"], + "ENCRYPTION_KEY": lf["LANGFUSE_ENCRYPTION_KEY"], + "NEXTAUTH_SECRET": lf["LANGFUSE_NEXTAUTH_SECRET"], + "CLICKHOUSE_PASSWORD": lf["LANGFUSE_CLICKHOUSE_PASSWORD"], + "REDIS_AUTH": lf["LANGFUSE_REDIS_PASSWORD"], + "LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY": lf["LANGFUSE_MINIO_PASSWORD"], + "LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY": lf["LANGFUSE_MINIO_PASSWORD"], + "LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY": lf["LANGFUSE_MINIO_PASSWORD"], + "LANGFUSE_INIT_PROJECT_PUBLIC_KEY": public_key, + "LANGFUSE_INIT_PROJECT_SECRET_KEY": secret_key, + }) + write_env(lf_path, lf, shell_safe=False) + + merged["LANGFUSE_PUBLIC_KEY"] = public_key + merged["LANGFUSE_SECRET_KEY"] = secret_key + write_env(Path(args.output), merged, shell_safe=True) + + preserved = sum(1 for k, v in base.items() if merged.get(k) == v) + changed = sorted(k for k, v in base.items() if merged.get(k) != v) + print(f"Base env: {base_path} ({len(base)} variables)") + print(f"Defaults: {defaults_path} ({len(defaults)} variables)") + if override_path: + print(f"Explicit override: {override_path} ({len(explicit)} variables)") + print(f"Runtime env written: {args.output} ({len(merged)} variables)") + print(f"Backend values preserved unchanged: {preserved}/{len(base)}") + if changed: + print("Backend values intentionally changed for OKE/container compatibility:") + for k in changed: + print(f" - {k}") + print(f"Langfuse env written: {args.langfuse_env}") + print("LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY synchronized automatically.") + +if __name__ == "__main__": + main() diff --git a/deploy/oke/load-test/scripts/prepare_env.sh b/deploy/oke/load-test/scripts/prepare_env.sh new file mode 100644 index 0000000..437a201 --- /dev/null +++ b/deploy/oke/load-test/scripts/prepare_env.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +BASE="$ROOT/templates/agent_template_backend/.env" +DEFAULTS="$LT/.env.loadtest.example" +OVERRIDE="$LT/.env.loadtest" + +[[ -f "$BASE" ]] || { echo "ERROR: backend env not found: $BASE" >&2; exit 1; } +[[ -f "$DEFAULTS" ]] || { echo "ERROR: load-test defaults not found: $DEFAULTS" >&2; exit 1; } + +args=( + --base "$BASE" + --overlay "$DEFAULTS" + --output "$LT/.env.runtime" + --langfuse-env "$LT/.env.langfuse" +) + +if [[ -f "$OVERRIDE" ]]; then + args+=(--override "$OVERRIDE") + echo "Using explicit load-test overrides: $OVERRIDE" +else + echo "No $OVERRIDE found; backend .env values will be preserved and example defaults only fill missing keys." +fi + +python3 "$LT/scripts/prepare_env.py" "${args[@]}" + +echo +echo "Runtime environment prepared: $LT/.env.runtime" +echo "Source of application configuration: $BASE" +echo "Optional explicit overrides: $OVERRIDE" diff --git a/deploy/oke/load-test/scripts/run_architecture_suite.sh b/deploy/oke/load-test/scripts/run_architecture_suite.sh new file mode 100644 index 0000000..07fd8c8 --- /dev/null +++ b/deploy/oke/load-test/scripts/run_architecture_suite.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +ENV="$LT/.env.runtime" +[[ -f "$ENV" ]] || "$LT/scripts/prepare_env.sh" +set -a; source "$ENV"; set +a + +run_case() { + local name="$1" rps="$2" dur="$3" scenario="$4" + echo "=== $name: ${rps} rps / ${dur} / ${scenario} ===" + python3 - "$ENV" "$rps" "$dur" "$scenario" <<'PY' +import sys +p,rps,dur,scenario=sys.argv[1:] +kv={} +order=[] +for line in open(p): + if '=' in line and not line.startswith('#'): + k,v=line.rstrip('\n').split('=',1); kv[k]=v; order.append(k) +kv['LOADTEST_RPS']=rps; kv['LOADTEST_DURATION']=dur; kv['LOADTEST_SCENARIO']=scenario +for k in ('LOADTEST_RPS','LOADTEST_DURATION','LOADTEST_SCENARIO'): + if k not in order: order.append(k) +with open(p,'w') as f: + f.write('# Generated/updated load-test runtime env. Do not commit.\n') + for k in order: f.write(f'{k}={kv[k]}\n') +PY + "$LT/scripts/run_load_test.sh" internal + kubectl -n "${K8S_NAMESPACE:-agent-load-test}" logs -f job/agent-load-generator || true + "$LT/scripts/collect_results.sh" +} + +run_case warmup 5 2m unique_sessions +run_case baseline 25 5m unique_sessions +run_case scale 100 10m unique_sessions +run_case shared-state 50 10m shared_sessions + +echo "Suite completed. Review deploy/oke/load-test/results and Langfuse." diff --git a/deploy/oke/load-test/scripts/run_load_test.sh b/deploy/oke/load-test/scripts/run_load_test.sh new file mode 100644 index 0000000..6242736 --- /dev/null +++ b/deploy/oke/load-test/scripts/run_load_test.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +ENV="$LT/.env.runtime" +[[ -f "$ENV" ]] || { echo "ERROR: $ENV not found" >&2; exit 1; } +source "$ENV" + +NS="${K8S_NAMESPACE:-agent-load-test}" +TARGET_MODE="${1:-${LOADTEST_TARGET_MODE:-internal}}" + +resolve_external_url() { + if [[ -n "${LOADTEST_EXTERNAL_URL:-}" ]]; then + printf '%s\n' "${LOADTEST_EXTERNAL_URL%/}" + return 0 + fi + + : "${EXISTING_LB_OCID:?Set EXISTING_LB_OCID or LOADTEST_EXTERNAL_URL in .env.runtime}" + + command -v oci >/dev/null 2>&1 || { + echo "ERROR: oci CLI not found and LOADTEST_EXTERNAL_URL is not set" >&2 + return 1 + } + + local oci_args=() + [[ -n "${OCI_CLI_PROFILE:-}" ]] && oci_args+=(--profile "$OCI_CLI_PROFILE") + + local ip + ip="$( + oci lb load-balancer get \ + --load-balancer-id "$EXISTING_LB_OCID" \ + "${oci_args[@]}" \ + --query 'data."ip-addresses"[0]."ip-address"' \ + --raw-output + )" + + [[ -n "$ip" && "$ip" != "null" ]] || { + echo "ERROR: could not resolve IP from existing LB $EXISTING_LB_OCID" >&2 + return 1 + } + + local proto="${BACKEND_LISTENER_PROTOCOL:-HTTP}" + local scheme + scheme="$(printf '%s' "$proto" | tr '[:upper:]' '[:lower:]')" + [[ "$scheme" == "http" || "$scheme" == "https" ]] || scheme="http" + + local port="${BACKEND_LISTENER_PORT:-8000}" + printf '%s://%s:%s\n' "$scheme" "$ip" "$port" +} + +case "$TARGET_MODE" in + external) + LOADTEST_TARGET="$(resolve_external_url)" + ;; + internal) + LOADTEST_TARGET="http://agent-template-backend.${NS}.svc.cluster.local:8000" + ;; + *) + echo "Usage: $0 [internal|external]" >&2 + exit 2 + ;; +esac + +echo "Load-test mode: $TARGET_MODE" +echo "Load-test target: $LOADTEST_TARGET" + +TMP_ENV="$(mktemp)" +trap 'rm -f "$TMP_ENV"' EXIT +cp "$ENV" "$TMP_ENV" + +python3 - "$TMP_ENV" "$LOADTEST_TARGET" "$TARGET_MODE" <<'PY' +import sys +p,target,mode=sys.argv[1:] +lines=[] +seen_target=False +seen_mode=False +for line in open(p): + if line.startswith("LOADTEST_TARGET="): + lines.append(f"LOADTEST_TARGET={target}\n") + seen_target=True + elif line.startswith("LOADTEST_TARGET_MODE="): + lines.append(f"LOADTEST_TARGET_MODE={mode}\n") + seen_mode=True + else: + lines.append(line) +if not seen_target: + lines.append(f"LOADTEST_TARGET={target}\n") +if not seen_mode: + lines.append(f"LOADTEST_TARGET_MODE={mode}\n") +open(p,"w").writelines(lines) +PY + +kubectl -n "$NS" create secret generic agent-backend-runtime \ + --from-env-file="$TMP_ENV" \ + --dry-run=client -o yaml | kubectl apply -f - + +kubectl -n "$NS" create configmap k6-load-script \ + --from-file=loadtest.js="$LT/loadgen/loadtest.js" \ + --dry-run=client -o yaml | kubectl apply -f - + +kubectl -n "$NS" delete job agent-load-generator \ + --ignore-not-found --wait=true + +sed \ + -e "s#namespace: agent-load-test#namespace: $NS#g" \ + "$LT/k8s/k6-job.yaml" | kubectl apply -f - + +echo "Load test started against $LOADTEST_TARGET" +echo "Follow: kubectl -n $NS logs -f job/agent-load-generator" +echo "Watch: watch kubectl -n $NS get pods,hpa" diff --git a/deploy/oke/load-test/scripts/set_llm_mode.sh b/deploy/oke/load-test/scripts/set_llm_mode.sh new file mode 100644 index 0000000..d9388ab --- /dev/null +++ b/deploy/oke/load-test/scripts/set_llm_mode.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +ENV="$ROOT/deploy/oke/load-test/.env.runtime" +MODE="${1:-}" +case "$MODE" in + mock) VALUE=mock ;; + real|oci) VALUE=oci_sdk ;; + *) echo "Usage: $0 mock|real"; exit 2 ;; +esac +python3 - "$ENV" "$VALUE" <<'PY' +import sys +p,v=sys.argv[1:] +lines=open(p).read().splitlines(); out=[]; found=False +for line in lines: + if line.startswith('LLM_PROVIDER='): + out.append('LLM_PROVIDER='+v); found=True + else: out.append(line) +if not found: out.append('LLM_PROVIDER='+v) +open(p,'w').write('\n'.join(out)+'\n') +PY +set -a; source "$ENV"; set +a +NS="${K8S_NAMESPACE:-agent-load-test}" +kubectl -n "$NS" create secret generic agent-backend-runtime --from-env-file="$ENV" --dry-run=client -o yaml | kubectl apply -f - +kubectl -n "$NS" rollout restart deployment/agent-template-backend +kubectl -n "$NS" rollout status deployment/agent-template-backend --timeout=10m +echo "LLM_PROVIDER=$VALUE" diff --git a/deploy/oke/load-test/scripts/smoke_test.sh b/deploy/oke/load-test/scripts/smoke_test.sh new file mode 100644 index 0000000..46937cd --- /dev/null +++ b/deploy/oke/load-test/scripts/smoke_test.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +ENV="$ROOT/deploy/oke/load-test/.env.runtime" +[[ -f "$ENV" ]] || { echo "ERROR: $ENV not found" >&2; exit 1; } +source "$ENV" + +NS="${K8S_NAMESPACE:-agent-load-test}" +MODE="${1:-external}" + +resolve_external_url() { + if [[ -n "${LOADTEST_EXTERNAL_URL:-}" ]]; then + printf '%s\n' "${LOADTEST_EXTERNAL_URL%/}" + return 0 + fi + + : "${EXISTING_LB_OCID:?Set EXISTING_LB_OCID or LOADTEST_EXTERNAL_URL in .env.runtime}" + + command -v oci >/dev/null 2>&1 || { + echo "ERROR: oci CLI not found and LOADTEST_EXTERNAL_URL is not set" >&2 + return 1 + } + + local oci_args=() + [[ -n "${OCI_CLI_PROFILE:-}" ]] && oci_args+=(--profile "$OCI_CLI_PROFILE") + + local ip + ip="$( + oci lb load-balancer get \ + --load-balancer-id "$EXISTING_LB_OCID" \ + "${oci_args[@]}" \ + --query 'data."ip-addresses"[0]."ip-address"' \ + --raw-output + )" + + [[ -n "$ip" && "$ip" != "null" ]] || { + echo "ERROR: could not resolve IP from existing LB $EXISTING_LB_OCID" >&2 + return 1 + } + + local proto="${BACKEND_LISTENER_PROTOCOL:-HTTP}" + local scheme + scheme="$(printf '%s' "$proto" | tr '[:upper:]' '[:lower:]')" + [[ "$scheme" == "http" || "$scheme" == "https" ]] || scheme="http" + + local port="${BACKEND_LISTENER_PORT:-8000}" + printf '%s://%s:%s\n' "$scheme" "$ip" "$port" +} + +if [[ "$MODE" == "external" ]]; then + URL="$(resolve_external_url)" +elif [[ "$MODE" == "internal" ]]; then + # Execute curl inside the cluster because *.svc.cluster.local is not resolvable + # from the developer workstation. + URL="http://agent-template-backend.${NS}.svc.cluster.local:8000" +else + echo "Usage: $0 [external|internal]" >&2 + exit 2 +fi + +SID="smoke-$(date +%s)" +PAYLOAD="{\"channel\":\"web\",\"agent_id\":\"${LOADTEST_AGENT_ID:-telecom_contas}\",\"tenant_id\":\"loadtest\",\"payload\":{\"text\":\"Olá. Responda apenas OK.\",\"session_id\":\"$SID\",\"user_id\":\"smoke\",\"customer_id\":\"smoke\",\"message_id\":\"smoke-1\"}}" + + +verify_langfuse_trace() { + [[ "${ENABLE_LANGFUSE:-true}" == "true" ]] || return 0 + local checkpod="langfuse-trace-check-$(date +%s)" + local needle="$SID" + echo "Verifying trace ingestion in Langfuse for session $needle ..." + kubectl -n "$NS" delete pod "$checkpod" --ignore-not-found >/dev/null 2>&1 || true + kubectl -n "$NS" run "$checkpod" --restart=Never --image=docker.io/curlimages/curl:latest \ + --overrides="$(cat </dev/null && { echo LANGFUSE_TRACE_OK; exit 0; }; i=\$((i+1)); sleep 5; done; echo LANGFUSE_TRACE_NOT_FOUND >&2; exit 1"]}]}} +JSON +)" >/dev/null + if ! kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$checkpod" --timeout=90s >/dev/null 2>&1; then + kubectl -n "$NS" logs "$checkpod" || true + kubectl -n "$NS" delete pod "$checkpod" --ignore-not-found >/dev/null 2>&1 || true + return 1 + fi + kubectl -n "$NS" logs "$checkpod" + kubectl -n "$NS" delete pod "$checkpod" --ignore-not-found >/dev/null +} + +echo "Smoke target: $URL ($MODE)" + +if [[ "$MODE" == "external" ]]; then + echo "Health:" + curl -fsS --connect-timeout 10 --max-time 30 "$URL/health" + echo + echo "Gateway message:" + curl -fsS --connect-timeout 10 --max-time 120 \ + -X POST "$URL/gateway/message" \ + -H 'Content-Type: application/json' \ + -H "X-Request-ID: smoke-$SID" \ + -d "$PAYLOAD" + echo +else + pod="backend-smoke-$(date +%s)" + kubectl -n "$NS" run "$pod" \ + --restart=Never \ + --image=docker.io/curlimages/curl:latest \ + --command -- sh -c \ + "set -e; + echo 'Health:'; + curl -fsS --connect-timeout 10 --max-time 30 '$URL/health'; + echo; + echo 'Gateway message:'; + curl -fsS --connect-timeout 10 --max-time 120 -X POST '$URL/gateway/message' \ + -H 'Content-Type: application/json' \ + -H 'X-Request-ID: smoke-$SID' \ + -d '$PAYLOAD'; + echo" >/dev/null + + if ! kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$pod" --timeout=180s >/dev/null 2>&1; then + kubectl -n "$NS" logs "$pod" || true + kubectl -n "$NS" describe pod "$pod" || true + kubectl -n "$NS" delete pod "$pod" --ignore-not-found >/dev/null 2>&1 || true + exit 1 + fi + + kubectl -n "$NS" logs "$pod" + kubectl -n "$NS" delete pod "$pod" --ignore-not-found >/dev/null +fi + +verify_langfuse_trace diff --git a/deploy/oke/load-test/scripts/start_langfuse_compose.sh b/deploy/oke/load-test/scripts/start_langfuse_compose.sh new file mode 100644 index 0000000..2835a53 --- /dev/null +++ b/deploy/oke/load-test/scripts/start_langfuse_compose.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +[[ -f "$LT/.env.langfuse" ]] || "$LT/scripts/prepare_env.sh" +docker compose \ + --env-file "$LT/.env.langfuse" \ + -f "$ROOT/libs/agent_framework/Infrastructure_Langfuse/docker-compose.yml" \ + up -d +echo "Local Langfuse: http://localhost:3005" +echo "Keys are synchronized in $LT/.env.langfuse and $LT/.env.runtime" diff --git a/deploy/oke/load-test/scripts/sync_langfuse_client_secret.sh b/deploy/oke/load-test/scripts/sync_langfuse_client_secret.sh new file mode 100644 index 0000000..0260eec --- /dev/null +++ b/deploy/oke/load-test/scripts/sync_langfuse_client_secret.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +RUNTIME_ENV="$LT/.env.runtime" +LF_ENV="$LT/.env.langfuse" + +log(){ printf '[langfuse-sync] %s\n' "$*"; } +die(){ printf '[langfuse-sync] ERROR: %s\n' "$*" >&2; exit 1; } + +# IMPORTANT: do not `source .env.runtime` here. It contains backend OCI settings +# (OCI_AUTH_MODE, OCI_PROFILE, OCI_CLI_PROFILE, etc.) that can change the OCI CLI +# exec-plugin environment used by kubectl/kubeconfig. +read_env_value() { + local file="$1" key="$2" default_value="${3:-}" + local value="" + if [[ -f "$file" ]]; then + value="$(sed -n -E "s/^${key}=(.*)$/\\1/p" "$file" | tail -n 1)" + value="${value%$'\r'}" + if [[ "$value" == \"*\" && "$value" == *\" ]]; then value="${value:1:${#value}-2}"; fi + if [[ "$value" == \'*\' && "$value" == *\' ]]; then value="${value:1:${#value}-2}"; fi + fi + printf '%s\n' "${value:-$default_value}" +} + +LF_NS="${LANGFUSE_NAMESPACE:-$(read_env_value "$RUNTIME_ENV" LANGFUSE_NAMESPACE langfuse)}" +APP_NS="${K8S_NAMESPACE:-$(read_env_value "$RUNTIME_ENV" K8S_NAMESPACE agent-load-test)}" +LF_SECRET="${LANGFUSE_RUNTIME_SECRET:-langfuse-runtime}" +CLIENT_SECRET="${LANGFUSE_CLIENT_SECRET:-langfuse-client}" +LF_HOST="${LANGFUSE_HOST:-$(read_env_value "$RUNTIME_ENV" LANGFUSE_HOST "http://langfuse-web.${LF_NS}.svc.cluster.local:3000")}" + +command -v kubectl >/dev/null 2>&1 || die "kubectl not found" + +log "Kubernetes context: $(kubectl config current-context 2>/dev/null || echo '')" +log "Langfuse namespace: $LF_NS" +log "Backend namespace: $APP_NS" + +# Do not hide kubectl errors. A credentials/context problem is not a missing Secret. +set +e +secret_err="$(mktemp)" +kubectl -n "$LF_NS" get secret "$LF_SECRET" >/dev/null 2>"$secret_err" +rc=$? +set -e +if (( rc != 0 )); then + msg="$(cat "$secret_err")" + rm -f "$secret_err" + echo "$msg" >&2 + echo >&2 + log "Secrets visible in namespace '$LF_NS':" + kubectl -n "$LF_NS" get secrets 2>&1 || true + echo >&2 + log "Langfuse web Secret references:" + kubectl -n "$LF_NS" get deployment langfuse-web \ + -o jsonpath='{.spec.template.spec.containers[0].envFrom[*].secretRef.name}{"\n"}' 2>&1 || true + die "cannot read Secret/$LF_SECRET in namespace $LF_NS (kubectl rc=$rc). See the kubectl error above; this may be authentication/context rather than a missing Secret." +fi +rm -f "$secret_err" + +get_secret(){ + local key="$1" + local encoded + encoded="$(kubectl -n "$LF_NS" get secret "$LF_SECRET" -o "jsonpath={.data.${key}}")" + [[ -n "$encoded" ]] && printf '%s' "$encoded" | base64 -d +} + +PUBLIC_KEY="$(get_secret LANGFUSE_INIT_PROJECT_PUBLIC_KEY || true)" +SECRET_KEY="$(get_secret LANGFUSE_INIT_PROJECT_SECRET_KEY || true)" +[[ -n "$PUBLIC_KEY" ]] || PUBLIC_KEY="$(get_secret LANGFUSE_PUBLIC_KEY || true)" +[[ -n "$SECRET_KEY" ]] || SECRET_KEY="$(get_secret LANGFUSE_SECRET_KEY || true)" + +# Last-resort local fallback: the same values generated by prepare_env.py. +if [[ -z "$PUBLIC_KEY" && -f "$LF_ENV" ]]; then + PUBLIC_KEY="$(read_env_value "$LF_ENV" LANGFUSE_INIT_PROJECT_PUBLIC_KEY '')" +fi +if [[ -z "$SECRET_KEY" && -f "$LF_ENV" ]]; then + SECRET_KEY="$(read_env_value "$LF_ENV" LANGFUSE_INIT_PROJECT_SECRET_KEY '')" +fi + +[[ -n "$PUBLIC_KEY" ]] || die "Langfuse public key is empty" +[[ -n "$SECRET_KEY" ]] || die "Langfuse secret key is empty" + +kubectl create namespace "$APP_NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null +kubectl -n "$APP_NS" create secret generic "$CLIENT_SECRET" \ + --from-literal=LANGFUSE_PUBLIC_KEY="$PUBLIC_KEY" \ + --from-literal=LANGFUSE_SECRET_KEY="$SECRET_KEY" \ + --from-literal=LANGFUSE_HOST="$LF_HOST" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + +log "Secret/$CLIENT_SECRET synchronized into namespace $APP_NS." +log "Public key: ${PUBLIC_KEY:0:10}... (secret key not printed)" diff --git a/deploy/oke/load-test/scripts/validate_dependencies.sh b/deploy/oke/load-test/scripts/validate_dependencies.sh new file mode 100644 index 0000000..be4ec74 --- /dev/null +++ b/deploy/oke/load-test/scripts/validate_dependencies.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +ENV="$LT/.env.runtime" +[[ -f "$ENV" ]] || { echo "Run prepare_env.sh first" >&2; exit 1; } +source "$ENV" +NS="${K8S_NAMESPACE:-agent-load-test}" +IMAGE="$(cat "$LT/.backend-image" 2>/dev/null || true)" +[[ -n "$IMAGE" ]] || IMAGE="${OCI_REGION_KEY}.ocir.io/${OCI_TENANCY_NAMESPACE}/${OCIR_REPOSITORY_PREFIX}/agent-template-backend:${IMAGE_TAG}" + +"$LT/scripts/sync_langfuse_client_secret.sh" + +kubectl -n "$NS" delete pod dependency-check --ignore-not-found >/dev/null 2>&1 || true +cat <")) + c = oracledb.connect( + user=os.environ["ADB_USER"], password=os.environ["ADB_PASSWORD"], + dsn=os.environ["ADB_DSN"], config_dir=os.environ["ADB_WALLET_LOCATION"], + wallet_location=os.environ["ADB_WALLET_LOCATION"], + wallet_password=os.getenv("ADB_WALLET_PASSWORD"), + ) + ok("oracle", c.version) + c.close() + + m = MongoClient(os.environ["MONGODB_URI"], serverSelectionTimeoutMS=10000) + ok("mongo", str(m.admin.command("ping"))) + m.close() + + host = os.environ["LANGFUSE_HOST"].rstrip("/") + health = urllib.request.urlopen(host + "/api/public/health", timeout=10) + ok("langfuse_health", str(health.status)) + + token = base64.b64encode((os.environ["LANGFUSE_PUBLIC_KEY"] + ":" + os.environ["LANGFUSE_SECRET_KEY"]).encode()).decode() + req = urllib.request.Request(host + "/api/public/projects", headers={"Authorization": "Basic " + token}) + with urllib.request.urlopen(req, timeout=15) as r: + body = r.read().decode("utf-8", errors="replace") + if r.status != 200: + raise RuntimeError(f"Langfuse authenticated API returned {r.status}") + json.loads(body) + ok("langfuse_authenticated_api", "200") + + ok("dependencies", "ALL_DEPENDENCIES_OK") + volumes: + - name: wallet + secret: + secretName: agent-backend-wallet +YAML + +if ! kubectl -n "$NS" wait --for=jsonpath='{.status.phase}'=Succeeded pod/dependency-check --timeout=180s >/dev/null 2>&1; then + kubectl -n "$NS" describe pod dependency-check || true + kubectl -n "$NS" logs dependency-check || true + exit 1 +fi +kubectl -n "$NS" logs dependency-check diff --git a/deploy/oke/load-test/scripts/validate_langfuse_integration.sh b/deploy/oke/load-test/scripts/validate_langfuse_integration.sh new file mode 100644 index 0000000..187e2af --- /dev/null +++ b/deploy/oke/load-test/scripts/validate_langfuse_integration.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +LT="$ROOT/deploy/oke/load-test" +RUNTIME_ENV="$LT/.env.runtime" + +log(){ printf '[langfuse-validate] %s\n' "$*"; } +die(){ printf '[langfuse-validate] ERROR: %s\n' "$*" >&2; exit 1; } + +# Do not source .env.runtime: OCI_* values can affect the OCI CLI exec plugin +# used by kubectl. Read only the values required by this validator. +read_env_value() { + local file="$1" key="$2" default_value="${3:-}" + local value="" + if [[ -f "$file" ]]; then + value="$(sed -n -E "s/^${key}=(.*)$/\\1/p" "$file" | tail -n 1)" + value="${value%$'\r'}" + if [[ "$value" == \"*\" && "$value" == *\" ]]; then value="${value:1:${#value}-2}"; fi + if [[ "$value" == \'*\' && "$value" == *\' ]]; then value="${value:1:${#value}-2}"; fi + fi + printf '%s\n' "${value:-$default_value}" +} + +LF_NS="${LANGFUSE_NAMESPACE:-$(read_env_value "$RUNTIME_ENV" LANGFUSE_NAMESPACE langfuse)}" +APP_NS="${K8S_NAMESPACE:-$(read_env_value "$RUNTIME_ENV" K8S_NAMESPACE agent-load-test)}" +CLIENT_SECRET="${LANGFUSE_CLIENT_SECRET:-langfuse-client}" +LF_HOST="${LANGFUSE_HOST:-$(read_env_value "$RUNTIME_ENV" LANGFUSE_HOST "http://langfuse-web.${LF_NS}.svc.cluster.local:3000")}" +POD="langfuse-auth-check-$(date +%s)" + +command -v kubectl >/dev/null 2>&1 || die "kubectl not found" + +log "Kubernetes context: $(kubectl config current-context 2>/dev/null || echo '')" + +# Fail early with a Kubernetes-auth specific message. +if ! kubectl auth can-i get secrets -n "$LF_NS" >/dev/null 2>&1; then + die "Kubernetes authentication/authorization failed. Confirm 'kubectl get nodes' works before validating Langfuse." +fi + +"$LT/scripts/sync_langfuse_client_secret.sh" + +cleanup(){ + kubectl -n "$APP_NS" delete pod "$POD" --ignore-not-found --wait=false >/dev/null 2>&1 || true +} +trap cleanup EXIT + +# Use a normal Pod manifest instead of `kubectl run --overrides`. +# This avoids JSON-patch escaping issues and keeps secretKeyRef explicit. +cat </dev/null +apiVersion: v1 +kind: Pod +metadata: + name: ${POD} + namespace: ${APP_NS} + labels: + app: langfuse-auth-check +spec: + restartPolicy: Never + containers: + - name: check + image: docker.io/curlimages/curl:latest + env: + - name: LANGFUSE_HOST + value: "${LF_HOST}" + - name: LANGFUSE_PUBLIC_KEY + valueFrom: + secretKeyRef: + name: ${CLIENT_SECRET} + key: LANGFUSE_PUBLIC_KEY + - name: LANGFUSE_SECRET_KEY + valueFrom: + secretKeyRef: + name: ${CLIENT_SECRET} + key: LANGFUSE_SECRET_KEY + command: ["sh", "-c"] + args: + - | + set -eu + test -n "\$LANGFUSE_PUBLIC_KEY" + test -n "\$LANGFUSE_SECRET_KEY" + + echo "Checking Langfuse health..." + curl -fsS "\$LANGFUSE_HOST/api/public/health" >/dev/null + + echo "Checking authenticated Langfuse Public API..." + code=\$(curl -sS -o /tmp/projects.json -w "%{http_code}" \ + -u "\$LANGFUSE_PUBLIC_KEY:\$LANGFUSE_SECRET_KEY" \ + "\$LANGFUSE_HOST/api/public/projects") + + if [ "\$code" != "200" ]; then + echo "LANGFUSE_AUTH_FAILED HTTP \$code" + cat /tmp/projects.json || true + exit 1 + fi + + echo "LANGFUSE_AUTH_OK" +YAML + +if ! kubectl -n "$APP_NS" wait --for=jsonpath='{.status.phase}'=Succeeded "pod/$POD" --timeout=120s >/dev/null 2>&1; then + kubectl -n "$APP_NS" logs "$POD" || true + kubectl -n "$APP_NS" describe pod "$POD" || true + exit 1 +fi + +kubectl -n "$APP_NS" logs "$POD" diff --git a/deploy/oke/load-test/scripts/watch_test.sh b/deploy/oke/load-test/scripts/watch_test.sh new file mode 100644 index 0000000..66b1fea --- /dev/null +++ b/deploy/oke/load-test/scripts/watch_test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +set -a; source "$ROOT/deploy/oke/load-test/.env.runtime"; set +a +NS="${K8S_NAMESPACE:-agent-load-test}" +while true; do + clear + date + echo '=== Pods ===' + kubectl -n "$NS" get pods -o wide + echo '=== HPA ===' + kubectl -n "$NS" get hpa + echo '=== Resource usage ===' + kubectl -n "$NS" top pods 2>/dev/null || echo 'metrics-server not available' + sleep 5 +done