From 8b3cc8dd10aec4facd3180ea9cfdad225ba1df36 Mon Sep 17 00:00:00 2001 From: Oracle Public Cloud User Date: Fri, 4 Sep 2026 13:54:16 +0000 Subject: [PATCH] adding files 2 --- .gitignore | 6 + OCI_DEVOPS_KAGENT_FUNCTION_FLOW.md | 205 +++ OCI_DEVOPS_REMEDIATION_PIPELINES.md | 225 +++ build_spec.yaml | 54 + docker/inventory-service.Dockerfile | 23 + docker/oke-remediator-job.Dockerfile | 13 + docker/oke-remediator-multi-check.Dockerfile | 13 + docker/order-service.Dockerfile | 23 + docker/payment-service.Dockerfile | 23 + inventory-service/pom.xml | 40 + .../InventoryServiceApplication.java | 13 + .../inventory/api/InventoryController.java | 40 + .../inventory/model/InventoryResponse.java | 5 + .../src/main/resources/application.yml | 21 + k8s/base/all.yaml | 4 + k8s/base/configmap.yaml | 14 + k8s/base/hpa.yaml | 22 + k8s/base/inventory.yaml | 48 + k8s/base/namespace.yaml | 5 + k8s/base/order-loadbalancer.yaml | 18 + k8s/base/order.yaml | 75 + k8s/base/payment.yaml | 41 + k8s/broken/business-failure-configmap.yaml | 14 + ...imagepull-secret-failure-command-spec.yaml | 48 + k8s/broken/order-hpa-limited.yaml | 22 + k8s/broken/payment-error.yaml | 41 + k8s/broken/revenue-risk-configmap.yaml | 14 + .../simulate-three-errors-command-spec.yaml | 58 + k8s/delete-application-command-spec.yaml | 145 ++ k8s/delete-namespace-command-spec.yaml | 50 + k8s/fix-business-config-command-spec.yaml | 87 ++ k8s/fix-imagepull-secret-command-spec.yaml | 66 + k8s/fix-order-hpa-command-spec.yaml | 44 + k8s/kagent-multi-error-payload.json | 25 + k8s/kagent/environment_variables.sh | 6 + k8s/kagent/kagent-mcp-lb.yaml | 28 + k8s/kagent/kagent-ui-lb-exported.yaml | 27 + k8s/oke-remediator-multi-check/README.md | 24 + k8s/oke-remediator-multi-check/cronjob.yaml | 50 + k8s/oke-remediator-multi-check/role.yaml | 30 + .../rolebinding.yaml | 13 + .../serviceaccount.yaml | 5 + k8s/oke-remediator/README.md | 50 + k8s/oke-remediator/cronjob.yaml | 56 + k8s/oke-remediator/role.yaml | 9 + k8s/oke-remediator/rolebinding.yaml | 13 + k8s/oke-remediator/serviceaccount.yaml | 5 + k8s/remediate-three-errors-command-spec.yaml | 94 ++ k8s/remediation/README.md | 71 + k8s/remediation/business-configmap.yaml | 14 + k8s/remediation/imagepull-all-services.yaml | 189 +++ k8s/remediation/imagepull-inventory.yaml | 53 + k8s/remediation/imagepull-order.yaml | 76 + k8s/remediation/imagepull-payment.yaml | 58 + k8s/remediation/order-hpa-capacity.yaml | 22 + k8s/remediation/order-rollout-restart.yaml | 77 + oke-remediator-multi-check/main.py | 1341 +++++++++++++++++ oke-remediator-multi-check/requirements.txt | 2 + order-service/pom.xml | 44 + .../demo/orders/OrderServiceApplication.java | 19 + .../demo/orders/api/OrderController.java | 41 + .../demo/orders/client/InventoryClient.java | 26 + .../demo/orders/client/PaymentClient.java | 29 + .../demo/orders/model/BusinessHealth.java | 15 + .../demo/orders/model/CreateOrderRequest.java | 13 + .../orders/model/InventoryReservation.java | 5 + .../oracle/demo/orders/model/OrderRecord.java | 15 + .../demo/orders/model/PaymentDecision.java | 5 + .../ObservabilityController.java | 107 ++ .../observability/ObservabilitySnapshot.java | 16 + .../orders/observability/ServiceSignal.java | 10 + .../demo/orders/service/OrderService.java | 157 ++ .../src/main/resources/application.yml | 28 + .../src/main/resources/static/app.js | 118 ++ .../src/main/resources/static/index.html | 106 ++ .../src/main/resources/static/styles.css | 273 ++++ payment-service/pom.xml | 40 + .../payment/PaymentServiceApplication.java | 13 + .../demo/payment/api/PaymentController.java | 44 + .../model/PaymentAuthorizationRequest.java | 5 + .../model/PaymentAuthorizationResponse.java | 5 + .../src/main/resources/application.yml | 21 + pom.xml | 32 + 83 files changed, 5045 insertions(+) create mode 100644 .gitignore create mode 100644 OCI_DEVOPS_KAGENT_FUNCTION_FLOW.md create mode 100644 OCI_DEVOPS_REMEDIATION_PIPELINES.md create mode 100644 build_spec.yaml create mode 100644 docker/inventory-service.Dockerfile create mode 100644 docker/oke-remediator-job.Dockerfile create mode 100644 docker/oke-remediator-multi-check.Dockerfile create mode 100644 docker/order-service.Dockerfile create mode 100644 docker/payment-service.Dockerfile create mode 100644 inventory-service/pom.xml create mode 100644 inventory-service/src/main/java/com/oracle/demo/inventory/InventoryServiceApplication.java create mode 100644 inventory-service/src/main/java/com/oracle/demo/inventory/api/InventoryController.java create mode 100644 inventory-service/src/main/java/com/oracle/demo/inventory/model/InventoryResponse.java create mode 100644 inventory-service/src/main/resources/application.yml create mode 100644 k8s/base/all.yaml create mode 100644 k8s/base/configmap.yaml create mode 100644 k8s/base/hpa.yaml create mode 100644 k8s/base/inventory.yaml create mode 100644 k8s/base/namespace.yaml create mode 100644 k8s/base/order-loadbalancer.yaml create mode 100644 k8s/base/order.yaml create mode 100644 k8s/base/payment.yaml create mode 100644 k8s/broken/business-failure-configmap.yaml create mode 100644 k8s/broken/imagepull-secret-failure-command-spec.yaml create mode 100644 k8s/broken/order-hpa-limited.yaml create mode 100644 k8s/broken/payment-error.yaml create mode 100644 k8s/broken/revenue-risk-configmap.yaml create mode 100644 k8s/broken/simulate-three-errors-command-spec.yaml create mode 100644 k8s/delete-application-command-spec.yaml create mode 100644 k8s/delete-namespace-command-spec.yaml create mode 100644 k8s/fix-business-config-command-spec.yaml create mode 100644 k8s/fix-imagepull-secret-command-spec.yaml create mode 100644 k8s/fix-order-hpa-command-spec.yaml create mode 100644 k8s/kagent-multi-error-payload.json create mode 100644 k8s/kagent/environment_variables.sh create mode 100644 k8s/kagent/kagent-mcp-lb.yaml create mode 100644 k8s/kagent/kagent-ui-lb-exported.yaml create mode 100644 k8s/oke-remediator-multi-check/README.md create mode 100644 k8s/oke-remediator-multi-check/cronjob.yaml create mode 100644 k8s/oke-remediator-multi-check/role.yaml create mode 100644 k8s/oke-remediator-multi-check/rolebinding.yaml create mode 100644 k8s/oke-remediator-multi-check/serviceaccount.yaml create mode 100644 k8s/oke-remediator/README.md create mode 100644 k8s/oke-remediator/cronjob.yaml create mode 100644 k8s/oke-remediator/role.yaml create mode 100644 k8s/oke-remediator/rolebinding.yaml create mode 100644 k8s/oke-remediator/serviceaccount.yaml create mode 100644 k8s/remediate-three-errors-command-spec.yaml create mode 100644 k8s/remediation/README.md create mode 100644 k8s/remediation/business-configmap.yaml create mode 100644 k8s/remediation/imagepull-all-services.yaml create mode 100644 k8s/remediation/imagepull-inventory.yaml create mode 100644 k8s/remediation/imagepull-order.yaml create mode 100644 k8s/remediation/imagepull-payment.yaml create mode 100644 k8s/remediation/order-hpa-capacity.yaml create mode 100644 k8s/remediation/order-rollout-restart.yaml create mode 100644 oke-remediator-multi-check/main.py create mode 100644 oke-remediator-multi-check/requirements.txt create mode 100644 order-service/pom.xml create mode 100644 order-service/src/main/java/com/oracle/demo/orders/OrderServiceApplication.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/api/OrderController.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/client/InventoryClient.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/client/PaymentClient.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/model/BusinessHealth.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/model/CreateOrderRequest.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/model/InventoryReservation.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/model/OrderRecord.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/model/PaymentDecision.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilityController.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilitySnapshot.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/observability/ServiceSignal.java create mode 100644 order-service/src/main/java/com/oracle/demo/orders/service/OrderService.java create mode 100644 order-service/src/main/resources/application.yml create mode 100644 order-service/src/main/resources/static/app.js create mode 100644 order-service/src/main/resources/static/index.html create mode 100644 order-service/src/main/resources/static/styles.css create mode 100644 payment-service/pom.xml create mode 100644 payment-service/src/main/java/com/oracle/demo/payment/PaymentServiceApplication.java create mode 100644 payment-service/src/main/java/com/oracle/demo/payment/api/PaymentController.java create mode 100644 payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationRequest.java create mode 100644 payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationResponse.java create mode 100644 payment-service/src/main/resources/application.yml create mode 100644 pom.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..00a0a81 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +target/ +.idea/ +.vscode/ +*.iml +*.log + diff --git a/OCI_DEVOPS_KAGENT_FUNCTION_FLOW.md b/OCI_DEVOPS_KAGENT_FUNCTION_FLOW.md new file mode 100644 index 0000000..709114d --- /dev/null +++ b/OCI_DEVOPS_KAGENT_FUNCTION_FLOW.md @@ -0,0 +1,205 @@ +# OCI DevOps + kagent + OCI Function Remediation Flow + +This runbook connects an OCI DevOps deployment pipeline with kagent through an OCI Function. + +## 1. Expose kagent MCP privately + +Edit the reserved private IP in: + +```text +k8s/kagent/kagent-controller-private-lb.yaml +``` + +Replace: + +```text + +``` + +with the OCI reserved private IP that belongs to the LoadBalancer subnet. + +Apply the private LoadBalancer service: + +```bash +kubectl apply -f k8s/kagent/kagent-controller-private-lb.yaml +kubectl get svc kagent-controller-private-lb -n kagent +``` + +Save the private IP and build the MCP URL: + +```text +http://:8083/mcp +``` + +The OCI Function subnet must be able to reach this private IP on TCP `8083`. + +If the Service was previously created without the reserved private IP annotation, delete and recreate it: + +```bash +kubectl delete svc kagent-controller-private-lb -n kagent +kubectl apply -f k8s/kagent/kagent-controller-private-lb.yaml +``` + +## 2. Deploy the OCI Function + +Function path: + +```text +oci-functions/kagent-devops-remediator +``` + +Configure these Function variables: + +```text +KAGENT_MCP_URL=http://:8083/mcp +KAGENT_AGENT_NAME=k8s-agent +KAGENT_AGENT_NAMESPACE=kagent +APP_NAMESPACE=kagent-demo +ORDER_OBSERVABILITY_URL=http:///api/observability +``` + +The Function calls kagent MCP tool `invoke_agent` and requests a strict JSON diagnosis. + +## 3. Add Function invocation to OCI DevOps + +In the deploy pipeline, add a stage after the OKE deployment: + +```text +Deploy application to OKE + -> Wait 60 seconds + -> Invoke Function: kagent-devops-remediator + -> Approval: review kagent diagnosis + -> Shell: apply ConfigMap fix + -> Shell: validate business health +``` + +The Function returns: + +```json +{ + "status": "HEALTHY|DEGRADED|FAILED", + "failureType": "BUSINESS_PROCESS_DEGRADATION", + "recommendedAction": "APPLY_CONFIGMAP_FIX", + "recommendedConfigMapData": { + "ORDER_MANUAL_REVIEW_RATE_THRESHOLD": "70", + "ORDER_REVENUE_AT_RISK_THRESHOLD": "15000" + }, + "deploymentsToRestart": ["order-service"] +} +``` + +## 4. Apply the ConfigMap remediation + +Use this command spec as a Shell deploy artifact: + +```text +k8s/fix-business-config-command-spec.yaml +``` + +It patches `demo-config` and restarts `order-service`. + +## 5. Validate the application + +Use this command spec after remediation: + +```text +k8s/validate-business-health-command-spec.yaml +``` + +Before using it, replace: + +```text +http:///api/observability +``` + +with the current LoadBalancer URL. + +## Recommended pipeline behavior + +Use this decision model: + +```text +CONTINUE -> no remediation +APPROVAL_REQUIRED -> pause for human review +APPLY_CONFIGMAP_FIX -> approval, then run fix-business-config-command-spec.yaml +ROLLBACK -> run rollback stage or redeploy previous image tag +REDEPLOY -> rerun deploy stage with corrected artifact +``` + +For the conference demo, keep approval before remediation. It makes the control boundary clear: kagent diagnoses, OCI DevOps executes. + +## Multi-error demo scenario + +This demo can simulate three simultaneous issues: + +```text +1. Business degradation: + ConfigMap thresholds make order-service report degraded business health. + +2. Image pull failure: + payment-service is patched with an invalid imagePullSecret. + +3. HPA ceiling: + order-service HPA is capped at 3 replicas. +``` + +Artifacts: + +```text +k8s/kagent-multi-error-payload.json +k8s/broken/simulate-three-errors-command-spec.yaml +k8s/remediate-three-errors-command-spec.yaml +k8s/fix-business-config-command-spec.yaml +k8s/fix-imagepull-secret-command-spec.yaml +k8s/fix-order-hpa-command-spec.yaml +``` + +Recommended OCI DevOps flow: + +```text +Deploy app + -> Wait 60-90 seconds + -> Optional Shell: simulate-three-errors-command-spec.yaml + -> Invoke Function with k8s/kagent-multi-error-payload.json + -> Approval: review kagent output + -> Shell: remediate-three-errors-command-spec.yaml + -> Shell: validate-business-health-command-spec.yaml + -> Invoke Function again to validate no active failures +``` + +The generic Function payload asks kagent to return a list of problems: + +```json +{ + "status": "HEALTHY|DEGRADED|FAILED", + "summary": "", + "problems": [ + { + "failureType": "BUSINESS_PROCESS_DEGRADATION|IMAGE_PULL_FAILURE|HPA_AUTOSCALING_FAILURE|NO_ACTIVE_FAILURE", + "rootCause": "", + "evidence": [], + "impact": "", + "recommendedAction": "CONTINUE|APPROVAL_REQUIRED|APPLY_CONFIGMAP_FIX|FIX_IMAGE_PULL_SECRET|PATCH_HPA|ROLLBACK|REDEPLOY", + "affectedResources": [], + "recommendedConfigMapData": {}, + "recommendedHpaPatch": {}, + "deploymentsToRestart": [] + } + ], + "overallRecommendedAction": "CONTINUE|APPROVAL_REQUIRED|APPLY_CONFIGMAP_FIX|FIX_IMAGE_PULL_SECRET|PATCH_HPA|ROLLBACK|REDEPLOY" +} +``` + +For a first demo, use one remediation stage: + +```text +remediate-three-errors-command-spec.yaml +``` + +For a more advanced demo, split remediation by action: + +```text +APPLY_CONFIGMAP_FIX -> fix-business-config-command-spec.yaml +FIX_IMAGE_PULL_SECRET -> fix-imagepull-secret-command-spec.yaml +PATCH_HPA -> fix-order-hpa-command-spec.yaml +``` diff --git a/OCI_DEVOPS_REMEDIATION_PIPELINES.md b/OCI_DEVOPS_REMEDIATION_PIPELINES.md new file mode 100644 index 0000000..b27ea2d --- /dev/null +++ b/OCI_DEVOPS_REMEDIATION_PIPELINES.md @@ -0,0 +1,225 @@ +# OCI DevOps Remediation Pipelines + +This demo uses one detection path and three independent remediation pipelines. The preferred remediation style is declarative Kubernetes manifests applied by OCI DevOps. + +## Detection flow + +Use the main deploy or diagnosis pipeline to invoke the kagent Function with: + +```text +k8s/kagent-multi-error-payload.json +``` + +kagent can return one or more problems: + +```text +BUSINESS_PROCESS_DEGRADATION +IMAGE_PULL_FAILURE +HPA_AUTOSCALING_FAILURE +``` + +OCI DevOps does not provide a simple native OR/branch based on the Function JSON. Use one of these patterns: + +```text +Manual demo: +Invoke Function -> Approval -> operator starts the correct remediation pipeline. + +Automated demo: +Function parses kagent response and calls OCI DevOps API to start the correct remediation pipeline. +``` + +## Pipeline 1: Business application remediation + +Pipeline name: + +```text +kagent-remediate-business-config +``` + +Purpose: + +```text +Fix order-service business degradation caused by ConfigMap thresholds or stuck review mode. +``` + +Artifacts: + +```text +Name: remediate-business-configmap +Type: Kubernetes manifest +Path: k8s/remediation/business-configmap.yaml + +Name: remediate-order-rollout-restart +Type: Kubernetes manifest +Path: k8s/remediation/order-rollout-restart.yaml +``` + +Recommended parameters: + +```text +APP_NAMESPACE=kagent-demo +CONFIGMAP_NAME=demo-config +ORDER_MANUAL_REVIEW_RATE_THRESHOLD=70 +ORDER_REVENUE_AT_RISK_THRESHOLD=15000 +ORDER_STUCK_REVIEW_MODE=false +PAYMENT_FORCE_TIMEOUT=false +INVENTORY_FORCE_OUTAGE=false +WAIT_TIMEOUT=300s +``` + +Stages: + +```text +approval-remediate-business + -> apply-business-configmap + -> apply-order-rollout-restart + -> wait-after-business-fix + -> invoke-kagent-function-validate +``` + +Important: + +```text +order-service reads ConfigMap values as environment variables. +After applying business-configmap.yaml, apply order-rollout-restart.yaml with a new . +``` + +## Pipeline 2: ImagePull remediation + +Pipeline name: + +```text +kagent-remediate-imagepull-secret +``` + +Purpose: + +```text +Restore the valid OCIR imagePullSecret in any business deployment: order-service, payment-service, or inventory-service. +``` + +Artifacts: + +```text +Name: remediate-imagepull-order +Type: Kubernetes manifest +Path: k8s/remediation/imagepull-order.yaml + +Name: remediate-imagepull-payment +Type: Kubernetes manifest +Path: k8s/remediation/imagepull-payment.yaml + +Name: remediate-imagepull-inventory +Type: Kubernetes manifest +Path: k8s/remediation/imagepull-inventory.yaml + +Name: remediate-imagepull-all-services +Type: Kubernetes manifest +Path: k8s/remediation/imagepull-all-services.yaml +``` + +Manifest placeholders: + +```text + + + + +``` + +Allowed values: + +```text +DEPLOYMENT_NAME=order-service +DEPLOYMENT_NAME=payment-service +DEPLOYMENT_NAME=inventory-service +``` + +When remediating another service, set `CONTAINER_NAME` to the same value as `DEPLOYMENT_NAME`. + +Stages: + +```text +approval-remediate-imagepull + -> apply-imagepull-service-manifest + -> wait-after-imagepull-fix + -> invoke-kagent-function-validate +``` + +For exactly three remediation pipelines, use `imagepull-all-services.yaml`. It restores the imagePullSecret in all three services and avoids conditional branching. + +Create one pipeline per target service only if you want narrower remediation: + +```text +kagent-remediate-imagepull-order +kagent-remediate-imagepull-payment +kagent-remediate-imagepull-inventory +``` + +Or create one pipeline and manually select the matching manifest artifact based on kagent output. + +## Pipeline 3: HPA capacity remediation + +Pipeline name: + +```text +kagent-remediate-order-hpa +``` + +Purpose: + +```text +Increase HPA capacity when order-service reaches maxReplicas and demand still requires more pods. +``` + +Artifact: + +```text +Name: remediate-order-hpa-capacity +Type: Kubernetes manifest +Path: k8s/remediation/order-hpa-capacity.yaml +``` + +Recommended parameters: + +```text +APP_NAMESPACE=kagent-demo +HPA_NAME=order-service +MIN_REPLICAS=3 +MAX_REPLICAS=8 +CPU_TARGET=65 +``` + +Stages: + +```text +approval-remediate-hpa + -> apply-order-hpa-capacity + -> wait-after-hpa-fix + -> invoke-kagent-function-validate +``` + +## Demo execution order + +Use this sequence for the conference: + +```text +1. Run kagent-deployment-pipeline to deploy the healthy application. +2. Run kagent-simulate-failures-pipeline to inject controlled failures. +3. Run invoke-kagent-function with kagent-multi-error-payload.json. +4. Review kagent output. +5. Run one or more remediation pipelines: + - kagent-remediate-business-config + - kagent-remediate-imagepull-order/payment/inventory + - kagent-remediate-order-hpa +6. Run final kagent validation. +``` + +## Mapping from kagent response to remediation pipeline + +```text +BUSINESS_PROCESS_DEGRADATION -> kagent-remediate-business-config +IMAGE_PULL_FAILURE -> kagent-remediate-imagepull-secret +HPA_AUTOSCALING_FAILURE -> kagent-remediate-order-hpa +NO_ACTIVE_FAILURE -> no remediation +``` diff --git a/build_spec.yaml b/build_spec.yaml new file mode 100644 index 0000000..02936f1 --- /dev/null +++ b/build_spec.yaml @@ -0,0 +1,54 @@ +version: 0.1 +component: build +timeoutInSeconds: 1800 +shell: bash + +steps: + - type: Command + name: Validate pipeline parameters + command: | + echo "REGISTRY=${REGISTRY}" + echo "NAMESPACE=${NAMESPACE}" + echo "IMAGE_TAG=${IMAGE_TAG}" + + if [ -z "${REGISTRY}" ] || [ -z "${NAMESPACE}" ] || [ -z "${IMAGE_TAG}" ]; then + echo "REGISTRY, NAMESPACE, and IMAGE_TAG are required" + exit 1 + fi + - type: Command + name: Login to container registry + command: | + docker login -u ${NAMESPACE}/${DOMAIN}/${USER} -p ${AUTHTOKEN} ${REGISTRY} + + - type: Command + name: Build container images + command: | + docker build -f docker/inventory-service.Dockerfile -t ${REGISTRY}/${NAMESPACE}/kagent-demo/inventory-service:${IMAGE_TAG} . + docker build -f docker/payment-service.Dockerfile -t ${REGISTRY}/${NAMESPACE}/kagent-demo/payment-service:${IMAGE_TAG} . + docker build -f docker/order-service.Dockerfile -t ${REGISTRY}/${NAMESPACE}/kagent-demo/order-service:${IMAGE_TAG} . + docker build -f docker/oke-remediator-multi-check.Dockerfile -t ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-multi-check:${IMAGE_TAG} . + + - type: Command + name: Push container images + command: | + docker push ${REGISTRY}/${NAMESPACE}/kagent-demo/inventory-service:${IMAGE_TAG} + docker push ${REGISTRY}/${NAMESPACE}/kagent-demo/payment-service:${IMAGE_TAG} + docker push ${REGISTRY}/${NAMESPACE}/kagent-demo/order-service:${IMAGE_TAG} + docker push ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-multi-check:${IMAGE_TAG} + +outputArtifacts: + - name: inventory-image + type: DOCKER_IMAGE + location: ${REGISTRY}/${NAMESPACE}/kagent-demo/inventory-service:${IMAGE_TAG} + + - name: payment-image + type: DOCKER_IMAGE + location: ${REGISTRY}/${NAMESPACE}/kagent-demo/payment-service:${IMAGE_TAG} + + - name: order-image + type: DOCKER_IMAGE + location: ${REGISTRY}/${NAMESPACE}/kagent-demo/order-service:${IMAGE_TAG} + + - name: oke-remediator-multi-image + type: DOCKER_IMAGE + location: ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-multi-check:${IMAGE_TAG} diff --git a/docker/inventory-service.Dockerfile b/docker/inventory-service.Dockerfile new file mode 100644 index 0000000..0e7d53d --- /dev/null +++ b/docker/inventory-service.Dockerfile @@ -0,0 +1,23 @@ +FROM container-registry.oracle.com/graalvm/jdk:25 AS build + +WORKDIR /workspace + +RUN microdnf install -y maven && microdnf clean all + +COPY pom.xml . +COPY payment-service/pom.xml payment-service/pom.xml +COPY order-service/pom.xml order-service/pom.xml +COPY inventory-service/pom.xml inventory-service/pom.xml +COPY inventory-service/src inventory-service/src + +RUN mvn -pl inventory-service -am clean package -DskipTests + +FROM container-registry.oracle.com/graalvm/jdk:25 + +WORKDIR /app + +COPY --from=build /workspace/inventory-service/target/inventory-service-1.0.0-SNAPSHOT.jar app.jar + +EXPOSE 8081 + +ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-jar", "app.jar"] diff --git a/docker/oke-remediator-job.Dockerfile b/docker/oke-remediator-job.Dockerfile new file mode 100644 index 0000000..6b240a1 --- /dev/null +++ b/docker/oke-remediator-job.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.11-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY oke-remediator-job/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY oke-remediator-job/main.py /app/main.py + +ENTRYPOINT ["python", "/app/main.py"] diff --git a/docker/oke-remediator-multi-check.Dockerfile b/docker/oke-remediator-multi-check.Dockerfile new file mode 100644 index 0000000..4a9c0e2 --- /dev/null +++ b/docker/oke-remediator-multi-check.Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.11-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 + +COPY oke-remediator-multi-check/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY oke-remediator-multi-check/main.py /app/main.py + +ENTRYPOINT ["python", "/app/main.py"] diff --git a/docker/order-service.Dockerfile b/docker/order-service.Dockerfile new file mode 100644 index 0000000..48c6fca --- /dev/null +++ b/docker/order-service.Dockerfile @@ -0,0 +1,23 @@ +FROM container-registry.oracle.com/graalvm/jdk:25 AS build + +WORKDIR /workspace + +RUN microdnf install -y maven && microdnf clean all + +COPY pom.xml . +COPY inventory-service/pom.xml inventory-service/pom.xml +COPY payment-service/pom.xml payment-service/pom.xml +COPY order-service/pom.xml order-service/pom.xml +COPY order-service/src order-service/src + +RUN mvn -pl order-service -am clean package -DskipTests + +FROM container-registry.oracle.com/graalvm/jdk:25 + +WORKDIR /app + +COPY --from=build /workspace/order-service/target/order-service-1.0.0-SNAPSHOT.jar app.jar + +EXPOSE 8080 + +ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-jar", "app.jar"] diff --git a/docker/payment-service.Dockerfile b/docker/payment-service.Dockerfile new file mode 100644 index 0000000..7539cc5 --- /dev/null +++ b/docker/payment-service.Dockerfile @@ -0,0 +1,23 @@ +FROM container-registry.oracle.com/graalvm/jdk:25 AS build + +WORKDIR /workspace + +RUN microdnf install -y maven && microdnf clean all + +COPY pom.xml . +COPY inventory-service/pom.xml inventory-service/pom.xml +COPY order-service/pom.xml order-service/pom.xml +COPY payment-service/pom.xml payment-service/pom.xml +COPY payment-service/src payment-service/src + +RUN mvn -pl payment-service -am clean package -DskipTests + +FROM container-registry.oracle.com/graalvm/jdk:25 + +WORKDIR /app + +COPY --from=build /workspace/payment-service/target/payment-service-1.0.0-SNAPSHOT.jar app.jar + +EXPOSE 8082 + +ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-jar", "app.jar"] diff --git a/inventory-service/pom.xml b/inventory-service/pom.xml new file mode 100644 index 0000000..c46a7fe --- /dev/null +++ b/inventory-service/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + + com.oracle.demo + kagent-oci-devops-demo + 1.0.0-SNAPSHOT + + + inventory-service + inventory-service + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-actuator + + + io.micrometer + micrometer-registry-prometheus + runtime + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + diff --git a/inventory-service/src/main/java/com/oracle/demo/inventory/InventoryServiceApplication.java b/inventory-service/src/main/java/com/oracle/demo/inventory/InventoryServiceApplication.java new file mode 100644 index 0000000..8fe08f0 --- /dev/null +++ b/inventory-service/src/main/java/com/oracle/demo/inventory/InventoryServiceApplication.java @@ -0,0 +1,13 @@ +package com.oracle.demo.inventory; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class InventoryServiceApplication { + + public static void main(String[] args) { + SpringApplication.run(InventoryServiceApplication.class, args); + } +} + diff --git a/inventory-service/src/main/java/com/oracle/demo/inventory/api/InventoryController.java b/inventory-service/src/main/java/com/oracle/demo/inventory/api/InventoryController.java new file mode 100644 index 0000000..8bd1c80 --- /dev/null +++ b/inventory-service/src/main/java/com/oracle/demo/inventory/api/InventoryController.java @@ -0,0 +1,40 @@ +package com.oracle.demo.inventory.api; + +import com.oracle.demo.inventory.model.InventoryResponse; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.Map; + +@RestController +@RequestMapping("/api/inventory") +public class InventoryController { + + private final Map stock = new HashMap<>(); + private final boolean forceOutage; + + public InventoryController(@Value("${demo.inventory.force-outage:false}") boolean forceOutage) { + this.forceOutage = forceOutage; + stock.put("LAPTOP-15", 25); + stock.put("MOUSE-WL", 100); + stock.put("MONITOR-27", 12); + } + + @GetMapping("/{sku}") + public InventoryResponse getInventory(@PathVariable String sku, @RequestParam(defaultValue = "1") int quantity) { + if (forceOutage) { + return new InventoryResponse(sku, false, 0, "Inventory service is intentionally degraded for the demo."); + } + + int available = stock.getOrDefault(sku, 0); + boolean reserved = available >= quantity; + String message = reserved ? "Stock reserved." : "Insufficient stock."; + return new InventoryResponse(sku, reserved, available, message); + } +} + diff --git a/inventory-service/src/main/java/com/oracle/demo/inventory/model/InventoryResponse.java b/inventory-service/src/main/java/com/oracle/demo/inventory/model/InventoryResponse.java new file mode 100644 index 0000000..c6b7c34 --- /dev/null +++ b/inventory-service/src/main/java/com/oracle/demo/inventory/model/InventoryResponse.java @@ -0,0 +1,5 @@ +package com.oracle.demo.inventory.model; + +public record InventoryResponse(String sku, boolean reserved, int availableUnits, String message) { +} + diff --git a/inventory-service/src/main/resources/application.yml b/inventory-service/src/main/resources/application.yml new file mode 100644 index 0000000..cffbbdf --- /dev/null +++ b/inventory-service/src/main/resources/application.yml @@ -0,0 +1,21 @@ +spring: + application: + name: inventory-service + +server: + port: 8081 + +management: + endpoints: + web: + exposure: + include: health,info,metrics,prometheus + endpoint: + health: + probes: + enabled: true + +demo: + inventory: + force-outage: false + diff --git a/k8s/base/all.yaml b/k8s/base/all.yaml new file mode 100644 index 0000000..a6ed70e --- /dev/null +++ b/k8s/base/all.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: List +items: [] + diff --git a/k8s/base/configmap.yaml b/k8s/base/configmap.yaml new file mode 100644 index 0000000..03c98e7 --- /dev/null +++ b/k8s/base/configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: demo-config + namespace: kagent-demo +data: + INVENTORY_FORCE_OUTAGE: "false" + PAYMENT_FORCE_TIMEOUT: "false" + PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999" + ORDER_STUCK_REVIEW_MODE: "false" + ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "40" + ORDER_REVENUE_AT_RISK_THRESHOLD: "5000" + INVENTORY_BASE_URL: "http://inventory-service:8081" + PAYMENT_BASE_URL: "http://payment-service:8082" diff --git a/k8s/base/hpa.yaml b/k8s/base/hpa.yaml new file mode 100644 index 0000000..6f8e34b --- /dev/null +++ b/k8s/base/hpa.yaml @@ -0,0 +1,22 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: order-service + namespace: kagent-demo + labels: + app: order-service + demo.oracle.com/remediation: hpa-capacity +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: order-service + minReplicas: 1 + maxReplicas: 1 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 30 diff --git a/k8s/base/inventory.yaml b/k8s/base/inventory.yaml new file mode 100644 index 0000000..1c87866 --- /dev/null +++ b/k8s/base/inventory.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inventory-service + namespace: kagent-demo +spec: + replicas: 1 + selector: + matchLabels: + app: inventory-service + template: + metadata: + labels: + app: inventory-service + spec: + imagePullSecrets: + - name: ocir-secret + containers: + - name: inventory-service + image: xxx.ocir.io/xxxxxxx/kagent-demo/inventory-service:2.0.0 + ports: + - containerPort: 8081 + env: + - name: DEMO_INVENTORY_FORCE_OUTAGE + valueFrom: + configMapKeyRef: + name: demo-config + key: INVENTORY_FORCE_OUTAGE + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8081 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8081 +--- +apiVersion: v1 +kind: Service +metadata: + name: inventory-service + namespace: kagent-demo +spec: + selector: + app: inventory-service + ports: + - port: 8081 + targetPort: 8081 diff --git a/k8s/base/namespace.yaml b/k8s/base/namespace.yaml new file mode 100644 index 0000000..5893b58 --- /dev/null +++ b/k8s/base/namespace.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kagent-demo + diff --git a/k8s/base/order-loadbalancer.yaml b/k8s/base/order-loadbalancer.yaml new file mode 100644 index 0000000..daddbde --- /dev/null +++ b/k8s/base/order-loadbalancer.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: order-service-lb + namespace: kagent-demo + annotations: + service.beta.kubernetes.io/oci-load-balancer-shape: flexible + service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" + service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "10" + oci.oraclecloud.com/reserved-ips: "40.233.17.25" +spec: + type: LoadBalancer + selector: + app: order-service + ports: + - name: http + port: 80 + targetPort: 8080 diff --git a/k8s/base/order.yaml b/k8s/base/order.yaml new file mode 100644 index 0000000..d8502eb --- /dev/null +++ b/k8s/base/order.yaml @@ -0,0 +1,75 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: order-service + namespace: kagent-demo +spec: + replicas: ${REPICLAS_ORDER} + selector: + matchLabels: + app: order-service + template: + metadata: + labels: + app: order-service + spec: + imagePullSecrets: + - name: ocir-secret + containers: + - name: order-service + image: ${REGISTRY}/${NAMESPACE}/kagent-demo/order-service:${IMAGE_TAG} + ports: + - containerPort: 8080 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi" + env: + - name: SERVICES_INVENTORY_BASE_URL + valueFrom: + configMapKeyRef: + name: demo-config + key: INVENTORY_BASE_URL + - name: SERVICES_PAYMENT_BASE_URL + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_BASE_URL + - name: DEMO_ORDERS_STUCK_REVIEW_MODE + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_STUCK_REVIEW_MODE + - name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD + - name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_REVENUE_AT_RISK_THRESHOLD + startupProbe: + httpGet: + path: /actuator/health/liveness + port: 8080 + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 3 \ No newline at end of file diff --git a/k8s/base/payment.yaml b/k8s/base/payment.yaml new file mode 100644 index 0000000..f242ca7 --- /dev/null +++ b/k8s/base/payment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payment-service + namespace: kagent-demo +spec: + replicas: 1 + selector: + matchLabels: + app: payment-service + template: + metadata: + labels: + app: payment-service + spec: + imagePullSecrets: + - name: ocir-secret + containers: + - name: payment-service + image: mty.ocir.io/xxxxxxx/kagent-demo/payment-service:2.0.0 + ports: + - containerPort: 8082 + env: + - name: DEMO_PAYMENT_FORCE_TIMEOUT + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_FORCE_TIMEOUT + - name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8082 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8082 diff --git a/k8s/broken/business-failure-configmap.yaml b/k8s/broken/business-failure-configmap.yaml new file mode 100644 index 0000000..890beb0 --- /dev/null +++ b/k8s/broken/business-failure-configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: demo-config + namespace: kagent-demo +data: + INVENTORY_FORCE_OUTAGE: "false" + PAYMENT_FORCE_TIMEOUT: "false" + PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999" + ORDER_STUCK_REVIEW_MODE: "true" + ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "40" + ORDER_REVENUE_AT_RISK_THRESHOLD: "5000" + INVENTORY_BASE_URL: "http://inventory-service:8081" + PAYMENT_BASE_URL: "http://payment-service:8082" diff --git a/k8s/broken/imagepull-secret-failure-command-spec.yaml b/k8s/broken/imagepull-secret-failure-command-spec.yaml new file mode 100644 index 0000000..b1cb424 --- /dev/null +++ b/k8s/broken/imagepull-secret-failure-command-spec.yaml @@ -0,0 +1,48 @@ +version: 0.1 +component: command +timeoutInSeconds: 600 +shell: bash +failImmediatelyOnError: true + +env: + variables: + APP_NAMESPACE: kagent-demo + DEPLOYMENT_NAME: payment-service + CONTAINER_NAME: payment-service + BROKEN_IMAGE_PULL_SECRET: missing-ocir-secret + +steps: + - type: Command + name: Simulate missing imagePullSecret + command: | + echo "Forcing ImagePullBackOff by replacing imagePullSecrets with ${BROKEN_IMAGE_PULL_SECRET}" + + kubectl patch deployment "${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" \ + --type merge \ + -p "{ + \"spec\": { + \"template\": { + \"spec\": { + \"imagePullSecrets\": [ + {\"name\": \"${BROKEN_IMAGE_PULL_SECRET}\"} + ], + \"containers\": [ + { + \"name\": \"${CONTAINER_NAME}\", + \"imagePullPolicy\": \"Always\" + } + ] + } + } + } + }" + + kubectl rollout restart deployment/"${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" + kubectl get pods -n "${APP_NAMESPACE}" -o wide + + - type: Command + name: Show image pull events + command: | + sleep 30 + kubectl get pods -n "${APP_NAMESPACE}" -o wide + kubectl get events -n "${APP_NAMESPACE}" --sort-by=.lastTimestamp | tail -40 diff --git a/k8s/broken/order-hpa-limited.yaml b/k8s/broken/order-hpa-limited.yaml new file mode 100644 index 0000000..084cbd8 --- /dev/null +++ b/k8s/broken/order-hpa-limited.yaml @@ -0,0 +1,22 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: order-service + namespace: kagent-demo + labels: + app: order-service + demo-scenario: hpa-limited-capacity +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: order-service + minReplicas: 2 + maxReplicas: 3 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 diff --git a/k8s/broken/payment-error.yaml b/k8s/broken/payment-error.yaml new file mode 100644 index 0000000..23d296c --- /dev/null +++ b/k8s/broken/payment-error.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payment-service + namespace: kagent-demo +spec: + replicas: 1 + selector: + matchLabels: + app: payment-service + template: + metadata: + labels: + app: payment-service + spec: + imagePullSecrets: + - name: ocir-secret-2 + containers: + - name: payment-service + image: mty.ocir.io/xxxxxxxx/kagent-demo/payment-service:latest + ports: + - containerPort: 8082 + env: + - name: DEMO_PAYMENT_FORCE_TIMEOUT + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_FORCE_TIMEOUT + - name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8082 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8082 diff --git a/k8s/broken/revenue-risk-configmap.yaml b/k8s/broken/revenue-risk-configmap.yaml new file mode 100644 index 0000000..cd76181 --- /dev/null +++ b/k8s/broken/revenue-risk-configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: demo-config + namespace: kagent-demo +data: + INVENTORY_FORCE_OUTAGE: "false" + PAYMENT_FORCE_TIMEOUT: "false" + PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999" + ORDER_STUCK_REVIEW_MODE: "false" + ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "30" + ORDER_REVENUE_AT_RISK_THRESHOLD: "5000" + INVENTORY_BASE_URL: "http://inventory-service:8081" + PAYMENT_BASE_URL: "http://payment-service:8082" diff --git a/k8s/broken/simulate-three-errors-command-spec.yaml b/k8s/broken/simulate-three-errors-command-spec.yaml new file mode 100644 index 0000000..0a2896b --- /dev/null +++ b/k8s/broken/simulate-three-errors-command-spec.yaml @@ -0,0 +1,58 @@ +version: 0.1 +component: command +timeoutInSeconds: 900 +shell: bash +failImmediatelyOnError: true + +env: + variables: + APP_NAMESPACE: kagent-demo + PAYMENT_DEPLOYMENT: payment-service + PAYMENT_CONTAINER: payment-service + BROKEN_IMAGE_PULL_SECRET: missing-ocir-secret + +steps: + - type: Command + name: Apply business degradation ConfigMap + command: | + kubectl apply -f k8s/broken/revenue-risk-configmap.yaml + kubectl rollout restart deployment/order-service -n "${APP_NAMESPACE}" + + - type: Command + name: Apply limited HPA + command: | + kubectl apply -f k8s/broken/order-hpa-limited.yaml + kubectl get hpa order-service -n "${APP_NAMESPACE}" -o wide + + - type: Command + name: Break payment image pull secret + command: | + kubectl patch deployment "${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" \ + --type merge \ + -p "{ + \"spec\": { + \"template\": { + \"spec\": { + \"imagePullSecrets\": [ + {\"name\": \"${BROKEN_IMAGE_PULL_SECRET}\"} + ], + \"containers\": [ + { + \"name\": \"${PAYMENT_CONTAINER}\", + \"imagePullPolicy\": \"Always\" + } + ] + } + } + } + }" + + kubectl rollout restart deployment/"${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" + + - type: Command + name: Show simulated failures + command: | + sleep 45 + kubectl get pods -n "${APP_NAMESPACE}" -o wide + kubectl get hpa -n "${APP_NAMESPACE}" -o wide + kubectl get events -n "${APP_NAMESPACE}" --sort-by=.lastTimestamp | tail -60 diff --git a/k8s/delete-application-command-spec.yaml b/k8s/delete-application-command-spec.yaml new file mode 100644 index 0000000..9751235 --- /dev/null +++ b/k8s/delete-application-command-spec.yaml @@ -0,0 +1,145 @@ +version: 0.1 +component: command +timeoutInSeconds: 900 +shell: bash +failImmediatelyOnError: true + +env: + variables: + APP_NAMESPACE: kagent-demo + WAIT_TIMEOUT: 300s + OCI_REGION: mx-monterrey-1 + OKE_CLUSTER_OCID: ocid1.cluster.oc1.mx-monterrey-1.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + OKE_ENDPOINT_TYPE: PUBLIC_ENDPOINT + +steps: + - type: Command + name: Validate parameters + command: | + if [ -z "${APP_NAMESPACE}" ]; then + echo "APP_NAMESPACE parameter is required" + exit 1 + fi + + if [ -z "${WAIT_TIMEOUT}" ]; then + echo "WAIT_TIMEOUT parameter is required" + exit 1 + fi + + if [ -z "${OCI_REGION}" ]; then + echo "OCI_REGION parameter is required" + exit 1 + fi + + if [ -z "${OKE_CLUSTER_OCID}" ] || [ "${OKE_CLUSTER_OCID}" = "" ]; then + echo "OKE_CLUSTER_OCID parameter is required" + exit 1 + fi + + if [ -z "${OKE_ENDPOINT_TYPE}" ]; then + echo "OKE_ENDPOINT_TYPE parameter is required" + exit 1 + fi + + echo "APP_NAMESPACE=${APP_NAMESPACE}" + echo "WAIT_TIMEOUT=${WAIT_TIMEOUT}" + echo "OCI_REGION=${OCI_REGION}" + echo "OKE_CLUSTER_OCID=${OKE_CLUSTER_OCID}" + echo "OKE_ENDPOINT_TYPE=${OKE_ENDPOINT_TYPE}" + + - type: Command + name: Configure kubeconfig + command: | + mkdir -p "${HOME}/.kube" + + oci ce cluster create-kubeconfig \ + --cluster-id "${OKE_CLUSTER_OCID}" \ + --file "${HOME}/.kube/config" \ + --region "${OCI_REGION}" \ + --token-version 2.0.0 \ + --kube-endpoint "${OKE_ENDPOINT_TYPE}" + + export KUBECONFIG="${HOME}/.kube/config" + echo "KUBECONFIG=${KUBECONFIG}" + + kubectl config current-context + kubectl cluster-info + + - type: Command + name: Show cleanup target + command: | + echo "Deleting kagent demo application resources" + kubectl version --client=true + kubectl get namespace "${APP_NAMESPACE}" + + - type: Command + name: Delete external entry point first + command: | + echo "Deleting LoadBalancer service first so OCI can release the public load balancer" + kubectl delete service order-service-lb \ + -n "${APP_NAMESPACE}" \ + --ignore-not-found=true + + - type: Command + name: Delete application workloads + command: | + kubectl delete deployment order-service inventory-service payment-service \ + -n "${APP_NAMESPACE}" \ + --ignore-not-found=true + + kubectl delete cronjob kagent-oke-remediator \ + -n "${APP_NAMESPACE}" \ + --ignore-not-found=true + + kubectl delete job -l cronjob-name=kagent-oke-remediator \ + -n "${APP_NAMESPACE}" \ + --ignore-not-found=true + + kubectl delete serviceaccount kagent-oke-remediator \ + -n "${APP_NAMESPACE}" \ + --ignore-not-found=true + + kubectl wait --for=delete pod \ + -l app=order-service \ + -n "${APP_NAMESPACE}" \ + --timeout="${WAIT_TIMEOUT}" || true + + kubectl wait --for=delete pod \ + -l app=inventory-service \ + -n "${APP_NAMESPACE}" \ + --timeout="${WAIT_TIMEOUT}" || true + + kubectl wait --for=delete pod \ + -l app=payment-service \ + -n "${APP_NAMESPACE}" \ + --timeout="${WAIT_TIMEOUT}" || true + + kubectl wait --for=delete job \ + -l cronjob-name=kagent-oke-remediator \ + -n "${APP_NAMESPACE}" \ + --timeout="${WAIT_TIMEOUT}" || true + + - type: Command + name: Delete internal services and app config + command: | + kubectl delete service order-service inventory-service payment-service \ + -n "${APP_NAMESPACE}" \ + --ignore-not-found=true + + kubectl delete configmap demo-config \ + -n "${APP_NAMESPACE}" \ + --ignore-not-found=true + + kubectl delete hpa order-service inventory-service payment-service \ + -n "${APP_NAMESPACE}" \ + --ignore-not-found=true + + - type: Command + name: Verify cleanup + command: | + echo "Remaining application resources in ${APP_NAMESPACE}:" + kubectl get deployment,svc,configmap,hpa,pod,cronjob,job,serviceaccount \ + -n "${APP_NAMESPACE}" \ + --ignore-not-found=true || true + + echo "Application cleanup completed. Namespace and imagePullSecrets were preserved." diff --git a/k8s/delete-namespace-command-spec.yaml b/k8s/delete-namespace-command-spec.yaml new file mode 100644 index 0000000..f94336d --- /dev/null +++ b/k8s/delete-namespace-command-spec.yaml @@ -0,0 +1,50 @@ +version: 0.1 +component: command +timeoutInSeconds: 900 +shell: bash +failImmediatelyOnError: true + +env: + variables: + APP_NAMESPACE: "kagent-demo" + WAIT_TIMEOUT: "300s" + +steps: + - type: Command + name: Confirm target namespace + command: | + echo "Full cleanup will delete the namespace and all resources inside it." + echo "APP_NAMESPACE=${APP_NAMESPACE}" + + if [ "${APP_NAMESPACE}" = "default" ] || [ "${APP_NAMESPACE}" = "kube-system" ] || [ "${APP_NAMESPACE}" = "kagent" ]; then + echo "Refusing to delete protected namespace: ${APP_NAMESPACE}" + exit 1 + fi + + kubectl get namespace "${APP_NAMESPACE}" + + - type: Command + name: Delete LoadBalancer before namespace cleanup + command: | + echo "Deleting LoadBalancer service first to start OCI load balancer cleanup" + kubectl delete service order-service-lb \ + -n "${APP_NAMESPACE}" \ + --ignore-not-found=true + + - type: Command + name: Delete namespace + command: | + kubectl delete namespace "${APP_NAMESPACE}" \ + --ignore-not-found=true \ + --timeout="${WAIT_TIMEOUT}" + + - type: Command + name: Verify namespace deletion + command: | + if kubectl get namespace "${APP_NAMESPACE}" >/dev/null 2>&1; then + echo "Namespace ${APP_NAMESPACE} still exists or is terminating." + kubectl get namespace "${APP_NAMESPACE}" -o wide + exit 1 + fi + + echo "Namespace ${APP_NAMESPACE} was deleted successfully." diff --git a/k8s/fix-business-config-command-spec.yaml b/k8s/fix-business-config-command-spec.yaml new file mode 100644 index 0000000..1df6ecd --- /dev/null +++ b/k8s/fix-business-config-command-spec.yaml @@ -0,0 +1,87 @@ +version: 0.1 +component: command +timeoutInSeconds: 900 +shell: bash +failImmediatelyOnError: true + +steps: + - type: Command + name: Validate remediation parameters + command: | + required_vars=( + APP_NAMESPACE + CONFIGMAP_NAME + OCI_REGION + OKE_CLUSTER_OCID + OKE_ENDPOINT_TYPE + ORDER_MANUAL_REVIEW_RATE_THRESHOLD + ORDER_REVENUE_AT_RISK_THRESHOLD + ORDER_STUCK_REVIEW_MODE + PAYMENT_FORCE_TIMEOUT + INVENTORY_FORCE_OUTAGE + WAIT_TIMEOUT + ) + + for var_name in "${required_vars[@]}"; do + if [ -z "${!var_name}" ]; then + echo "Required parameter is missing: ${var_name}" + exit 1 + fi + done + + echo "APP_NAMESPACE=${APP_NAMESPACE}" + echo "CONFIGMAP_NAME=${CONFIGMAP_NAME}" + echo "OCI_REGION=${OCI_REGION}" + echo "OKE_CLUSTER_OCID=${OKE_CLUSTER_OCID}" + echo "OKE_ENDPOINT_TYPE=${OKE_ENDPOINT_TYPE}" + echo "ORDER_MANUAL_REVIEW_RATE_THRESHOLD=${ORDER_MANUAL_REVIEW_RATE_THRESHOLD}" + echo "ORDER_REVENUE_AT_RISK_THRESHOLD=${ORDER_REVENUE_AT_RISK_THRESHOLD}" + echo "ORDER_STUCK_REVIEW_MODE=${ORDER_STUCK_REVIEW_MODE}" + echo "PAYMENT_FORCE_TIMEOUT=${PAYMENT_FORCE_TIMEOUT}" + echo "INVENTORY_FORCE_OUTAGE=${INVENTORY_FORCE_OUTAGE}" + echo "WAIT_TIMEOUT=${WAIT_TIMEOUT}" + + - type: Command + name: Configure kubeconfig + command: | + mkdir -p "${HOME}/.kube" + + oci ce cluster create-kubeconfig \ + --cluster-id "${OKE_CLUSTER_OCID}" \ + --file "${HOME}/.kube/config" \ + --region "${OCI_REGION}" \ + --token-version 2.0.0 \ + --kube-endpoint "${OKE_ENDPOINT_TYPE}" + + export KUBECONFIG="${HOME}/.kube/config" + echo "KUBECONFIG=${KUBECONFIG}" + + kubectl config current-context + kubectl cluster-info + + - type: Command + name: Apply business ConfigMap remediation + command: | + kubectl patch configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" \ + --type merge \ + -p "{ + \"data\": { + \"ORDER_MANUAL_REVIEW_RATE_THRESHOLD\": \"${ORDER_MANUAL_REVIEW_RATE_THRESHOLD}\", + \"ORDER_REVENUE_AT_RISK_THRESHOLD\": \"${ORDER_REVENUE_AT_RISK_THRESHOLD}\", + \"ORDER_STUCK_REVIEW_MODE\": \"${ORDER_STUCK_REVIEW_MODE}\", + \"PAYMENT_FORCE_TIMEOUT\": \"${PAYMENT_FORCE_TIMEOUT}\", + \"INVENTORY_FORCE_OUTAGE\": \"${INVENTORY_FORCE_OUTAGE}\" + } + }" + + - type: Command + name: Restart order service + command: | + kubectl rollout restart deployment/order-service -n "${APP_NAMESPACE}" + kubectl rollout status deployment/order-service -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}" + + - type: Command + name: Show remediated configuration + command: | + kubectl get configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" -o yaml + kubectl get pods -n "${APP_NAMESPACE}" -o wide diff --git a/k8s/fix-imagepull-secret-command-spec.yaml b/k8s/fix-imagepull-secret-command-spec.yaml new file mode 100644 index 0000000..ec5e9b6 --- /dev/null +++ b/k8s/fix-imagepull-secret-command-spec.yaml @@ -0,0 +1,66 @@ +version: 0.1 +component: command +timeoutInSeconds: 900 +shell: bash +failImmediatelyOnError: true + +env: + variables: + APP_NAMESPACE: kagent-demo + DEPLOYMENT_NAME: payment-service + CONTAINER_NAME: payment-service + IMAGE_PULL_SECRET: ocir-secret + WAIT_TIMEOUT: 300s + +steps: + - type: Command + name: Restore valid imagePullSecret + command: | + case "${DEPLOYMENT_NAME}" in + order-service|payment-service|inventory-service) + echo "Target deployment is valid: ${DEPLOYMENT_NAME}" + ;; + *) + echo "Invalid DEPLOYMENT_NAME=${DEPLOYMENT_NAME}" + echo "Allowed values: order-service, payment-service, inventory-service" + exit 1 + ;; + esac + + if [ -z "${CONTAINER_NAME}" ]; then + CONTAINER_NAME="${DEPLOYMENT_NAME}" + fi + + echo "APP_NAMESPACE=${APP_NAMESPACE}" + echo "DEPLOYMENT_NAME=${DEPLOYMENT_NAME}" + echo "CONTAINER_NAME=${CONTAINER_NAME}" + echo "IMAGE_PULL_SECRET=${IMAGE_PULL_SECRET}" + + kubectl patch deployment "${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" \ + --type merge \ + -p "{ + \"spec\": { + \"template\": { + \"spec\": { + \"imagePullSecrets\": [ + {\"name\": \"${IMAGE_PULL_SECRET}\"} + ], + \"containers\": [ + { + \"name\": \"${CONTAINER_NAME}\", + \"imagePullPolicy\": \"IfNotPresent\" + } + ] + } + } + } + }" + + kubectl rollout restart deployment/"${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" + kubectl rollout status deployment/"${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}" + + - type: Command + name: Verify restored image pull + command: | + kubectl get deployment "${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" -o jsonpath='{.spec.template.spec.imagePullSecrets}{"\n"}' + kubectl get pods -n "${APP_NAMESPACE}" -o wide diff --git a/k8s/fix-order-hpa-command-spec.yaml b/k8s/fix-order-hpa-command-spec.yaml new file mode 100644 index 0000000..3041985 --- /dev/null +++ b/k8s/fix-order-hpa-command-spec.yaml @@ -0,0 +1,44 @@ +version: 0.1 +component: command +timeoutInSeconds: 600 +shell: bash +failImmediatelyOnError: true + +env: + variables: + APP_NAMESPACE: kagent-demo + HPA_NAME: order-service + MIN_REPLICAS: "3" + MAX_REPLICAS: "8" + CPU_TARGET: "65" + +steps: + - type: Command + name: Patch order HPA capacity + command: | + kubectl patch hpa "${HPA_NAME}" -n "${APP_NAMESPACE}" \ + --type merge \ + -p "{ + \"spec\": { + \"minReplicas\": ${MIN_REPLICAS}, + \"maxReplicas\": ${MAX_REPLICAS}, + \"metrics\": [ + { + \"type\": \"Resource\", + \"resource\": { + \"name\": \"cpu\", + \"target\": { + \"type\": \"Utilization\", + \"averageUtilization\": ${CPU_TARGET} + } + } + } + ] + } + }" + + - type: Command + name: Show HPA state + command: | + kubectl get hpa "${HPA_NAME}" -n "${APP_NAMESPACE}" -o wide + kubectl describe hpa "${HPA_NAME}" -n "${APP_NAMESPACE}" diff --git a/k8s/kagent-multi-error-payload.json b/k8s/kagent-multi-error-payload.json new file mode 100644 index 0000000..3a2f333 --- /dev/null +++ b/k8s/kagent-multi-error-payload.json @@ -0,0 +1,25 @@ +{ + "responseMode": "json", + "region": "${REGION}", + "appNamespace": "${APP_NAMESPACE}", + "observabilityUrl": "${ORDER_OBSERVABILITY_URL}", + "notificationTopicOcid": "${NOTIFICATION_TOPIC_OCID}", + "remediationPipelines": { + "business": { + "approvalUrl": "${BUSINESS_REMEDIATION_APPROVAL_URL}", + "projectOcid": "${BUSINESS_REMEDIATION_PROJECT_OCID}", + "pipelineOcid": "${BUSINESS_REMEDIATION_PIPELINE_OCID}" + }, + "imagePull": { + "approvalUrl": "${IMAGEPULL_REMEDIATION_APPROVAL_URL}", + "projectOcid": "${IMAGEPULL_REMEDIATION_PROJECT_OCID}", + "pipelineOcid": "${IMAGEPULL_REMEDIATION_PIPELINE_OCID}" + }, + "hpa": { + "approvalUrl": "${HPA_REMEDIATION_APPROVAL_URL}", + "projectOcid": "${HPA_REMEDIATION_PROJECT_OCID}", + "pipelineOcid": "${HPA_REMEDIATION_PIPELINE_OCID}" + } + }, + "prompt": "Analiza el namespace kagent-demo despues del despliegue OCI DevOps. La URL de observabilidad obligatoria esta en el campo observabilityUrl del payload. Debes consultarla antes de responder y no debes preguntar por esa URL. Busca fallas activas, no eventos historicos, en estas categorias: BUSINESS_PROCESS_DEGRADATION, IMAGE_PULL_FAILURE, CRASH_LOOP_FAILURE, PROBE_FAILURE, HPA_AUTOSCALING_FAILURE, POD_SCHEDULING_FAILURE, DEPENDENCY_FAILURE, OKE_PLATFORM_FAILURE y NO_ACTIVE_FAILURE. Revisa pods, deployments, services, endpoints, eventos recientes, ConfigMap demo-config, imagePullSecrets, service accounts, HPA, logs recientes de order-service, inventory-service y payment-service, y el dashboard/API de observabilidad. Para degradacion de negocio, usa la respuesta del dashboard/API como fuente de verdad: si businessHealth.healthy=false, services incluye order-service con status DEGRADED, manualReviewRate supera el umbral configurado, pendingReviewOrders es alto o revenueAtRisk supera ORDER_REVENUE_AT_RISK_THRESHOLD, debes clasificar BUSINESS_PROCESS_DEGRADATION aunque todos los pods esten Running. Detecta si existen uno o varios problemas al mismo tiempo. Para cada problema, entrega evidencia concreta, causa raiz probable, impacto y accion recomendada para OCI DevOps. Si hay degradacion de negocio por cardToken con prefijo 9999, explica que PAYMENT_MANUAL_REVIEW_CARD_PREFIX envia esas ordenes a PENDING_REVIEW y que ORDER_MANUAL_REVIEW_RATE_THRESHOLD y ORDER_REVENUE_AT_RISK_THRESHOLD definen cuando order-service pasa a DEGRADED. Si hay degradacion de negocio, recomienda valores corregidos del ConfigMap. Si hay ImagePullBackOff o ErrImagePull, identifica deployment, imagen afectada y secreto faltante o invalido. Si hay HPA limitado, identifica si desired replicas alcanzo maxReplicas y recomienda nuevo minReplicas, maxReplicas y target CPU. Incluye un campo report para correo con title, executiveSummary, rootCause, businessImpact, evidence, configMapAnalysis y recommendedAction. El report debe ser entendible por un aprobador de OCI DevOps y debe explicar claramente por que la aplicacion esta degradada y que pipeline de remediacion debe aprobar. Responde exclusivamente JSON puro, sin markdown, sin texto antes ni despues, con este formato: {\"status\":\"HEALTHY|DEGRADED|FAILED\",\"summary\":\"\",\"report\":{\"title\":\"\",\"executiveSummary\":\"\",\"rootCause\":\"\",\"businessImpact\":\"\",\"evidence\":[],\"configMapAnalysis\":\"\",\"recommendedAction\":\"\"},\"problems\":[{\"failureType\":\"BUSINESS_PROCESS_DEGRADATION|IMAGE_PULL_FAILURE|HPA_AUTOSCALING_FAILURE|NO_ACTIVE_FAILURE\",\"rootCause\":\"\",\"evidence\":[],\"impact\":\"\",\"recommendedAction\":\"CONTINUE|APPROVAL_REQUIRED|APPLY_CONFIGMAP_FIX|FIX_IMAGE_PULL_SECRET|PATCH_HPA|ROLLBACK|REDEPLOY\",\"affectedResources\":[],\"recommendedConfigMapData\":{},\"recommendedHpaPatch\":{},\"deploymentsToRestart\":[]}],\"overallRecommendedAction\":\"CONTINUE|APPROVAL_REQUIRED|APPLY_CONFIGMAP_FIX|FIX_IMAGE_PULL_SECRET|PATCH_HPA|ROLLBACK|REDEPLOY\"}" +} diff --git a/k8s/kagent/environment_variables.sh b/k8s/kagent/environment_variables.sh new file mode 100644 index 0000000..097513d --- /dev/null +++ b/k8s/kagent/environment_variables.sh @@ -0,0 +1,6 @@ +export OCI_GENAI_API_KEY="sk-eavZt2RSpxxxxxxxxxxxxxxxxxxxxxxxxx" +export OCI_GENAI_REGION="us-chicago-1" +export OCI_GENAI_BASE_URL="https://inference.generativeai.${OCI_GENAI_REGION}.oci.oraclecloud.com/openai/v1" +export OCI_GENAI_MODEL="openai.gpt-4o" +export OCI_GENAI_PROJECT_OCID="ocid1.generativeaiproject.oc1.us-chicago-1.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + diff --git a/k8s/kagent/kagent-mcp-lb.yaml b/k8s/kagent/kagent-mcp-lb.yaml new file mode 100644 index 0000000..72153c0 --- /dev/null +++ b/k8s/kagent/kagent-mcp-lb.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Service +metadata: + name: kagent-controller-lb + namespace: kagent + annotations: + oci.oraclecloud.com/load-balancer-type: "nlb" + oci.oraclecloud.com/reserved-ips: "40.233.18.237" + labels: + app.kubernetes.io/instance: kagent + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: kagent + app.kubernetes.io/part-of: kagent + app.kubernetes.io/version: 0.9.12 + helm.sh/chart: kagent-0.9.12 +spec: + type: LoadBalancer + externalTrafficPolicy: Cluster + ports: + - name: mcp + port: 8083 + protocol: TCP + targetPort: 8083 + nodePort: 31083 + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: kagent + app.kubernetes.io/name: kagent \ No newline at end of file diff --git a/k8s/kagent/kagent-ui-lb-exported.yaml b/k8s/kagent/kagent-ui-lb-exported.yaml new file mode 100644 index 0000000..8267a2f --- /dev/null +++ b/k8s/kagent/kagent-ui-lb-exported.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + name: kagent-ui-lb + namespace: kagent + annotations: + oci.oraclecloud.com/load-balancer-type: "nlb" + oci.oraclecloud.com/reserved-ips: "40.233.25.148" + labels: + app.kubernetes.io/instance: kagent + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: kagent + app.kubernetes.io/part-of: kagent + app.kubernetes.io/version: 0.9.12 + helm.sh/chart: kagent-0.9.12 +spec: + type: LoadBalancer + externalTrafficPolicy: Cluster + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + nodePort: 31002 + selector: + app.kubernetes.io/component: ui + app.kubernetes.io/instance: kagent + app.kubernetes.io/name: kagent diff --git a/k8s/oke-remediator-multi-check/README.md b/k8s/oke-remediator-multi-check/README.md new file mode 100644 index 0000000..63f9983 --- /dev/null +++ b/k8s/oke-remediator-multi-check/README.md @@ -0,0 +1,24 @@ +# OKE Remediator Multi-Check + +This workload runs every 3 minutes and performs three independent kagent checks as separate MCP interactions: + +1. Business application health for Orders +2. HPA autoscaling analysis for `order-service` +3. ImagePull analysis for application deployments + +Each result is normalized to a structured JSON shape. The worker stores one state entry per `error_code` in a Kubernetes ConfigMap and only sends one email plus one remediation deployment trigger per active incident signature. + +## Deploy + +```bash +kubectl apply -f k8s/oke-remediator-multi-check/serviceaccount.yaml +kubectl apply -f k8s/oke-remediator-multi-check/role.yaml +kubectl apply -f k8s/oke-remediator-multi-check/rolebinding.yaml +kubectl apply -f k8s/oke-remediator-multi-check/cronjob.yaml +``` + +## Notes + +- The remediation pipelines must include approval stages. +- The notification email includes the remediation deployment approval link when the deployment is created successfully. +- The worker is intended for OKE Workload Identity. diff --git a/k8s/oke-remediator-multi-check/cronjob.yaml b/k8s/oke-remediator-multi-check/cronjob.yaml new file mode 100644 index 0000000..00b716a --- /dev/null +++ b/k8s/oke-remediator-multi-check/cronjob.yaml @@ -0,0 +1,50 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: kagent-oke-remediator-multi + namespace: kagent-demo +spec: + schedule: "*/3 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 1 + template: + spec: + restartPolicy: Never + serviceAccountName: kagent-oke-remediator-multi + imagePullSecrets: + - name: ocir-secret + containers: + - name: kagent-oke-remediator-multi + image: ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-multi-check:${IMAGE_TAG} + imagePullPolicy: Always + env: + - name: OCI_REGION + value: mx-monterrey-1 + - name: APP_NAMESPACE + value: kagent-demo + - name: KAGENT_MCP_URL + value: http://kagent-controller.kagent.svc.cluster.local:8083/mcp + - name: KAGENT_AGENT_NAME + value: k8s-agent + - name: KAGENT_AGENT_NAMESPACE + value: kagent + - name: ORDER_OBSERVABILITY_URL + value: http://order-service-lb.kagent-demo.svc.cluster.local/api/observability + - name: NOTIFICATION_TOPIC_OCID + value: ${NOTIFICATION_TOPIC_OCID} + - name: BUSINESS_REMEDIATION_PROJECT_OCID + value: ${BUSINESS_REMEDIATION_PROJECT_OCID} + - name: BUSINESS_REMEDIATION_PIPELINE_OCID + value: ${BUSINESS_REMEDIATION_PIPELINE_OCID} + - name: IMAGEPULL_REMEDIATION_PROJECT_OCID + value: ${IMAGEPULL_REMEDIATION_PROJECT_OCID} + - name: IMAGEPULL_REMEDIATION_PIPELINE_OCID + value: ${IMAGEPULL_REMEDIATION_PIPELINE_OCID} + - name: HPA_REMEDIATION_PROJECT_OCID + value: ${HPA_REMEDIATION_PROJECT_OCID} + - name: HPA_REMEDIATION_PIPELINE_OCID + value: ${HPA_REMEDIATION_PIPELINE_OCID} diff --git a/k8s/oke-remediator-multi-check/role.yaml b/k8s/oke-remediator-multi-check/role.yaml new file mode 100644 index 0000000..c8e1e35 --- /dev/null +++ b/k8s/oke-remediator-multi-check/role.yaml @@ -0,0 +1,30 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kagent-oke-remediator-multi + namespace: kagent-demo +rules: + - apiGroups: [""] + resources: + - pods + - events + verbs: + - get + - list + - watch + - apiGroups: ["apps"] + resources: + - deployments + - replicasets + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - create + - update + - patch diff --git a/k8s/oke-remediator-multi-check/rolebinding.yaml b/k8s/oke-remediator-multi-check/rolebinding.yaml new file mode 100644 index 0000000..187c2a2 --- /dev/null +++ b/k8s/oke-remediator-multi-check/rolebinding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kagent-oke-remediator-multi + namespace: kagent-demo +subjects: + - kind: ServiceAccount + name: kagent-oke-remediator-multi + namespace: kagent-demo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kagent-oke-remediator-multi diff --git a/k8s/oke-remediator-multi-check/serviceaccount.yaml b/k8s/oke-remediator-multi-check/serviceaccount.yaml new file mode 100644 index 0000000..293d773 --- /dev/null +++ b/k8s/oke-remediator-multi-check/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kagent-oke-remediator-multi + namespace: kagent-demo diff --git a/k8s/oke-remediator/README.md b/k8s/oke-remediator/README.md new file mode 100644 index 0000000..91f3b1d --- /dev/null +++ b/k8s/oke-remediator/README.md @@ -0,0 +1,50 @@ +# OKE Remediator Job + +This workload runs inside OKE as a `CronJob` and performs the same high-level flow as the OCI Function: + +1. Queries the business observability API. +2. Sends a task to `kagent` through MCP. +3. Detects business degradation, image pull failures, or HPA capacity issues. +4. Triggers the corresponding OCI DevOps remediation deployment for approval. +5. Sends an OCI Notifications email with the diagnostic report and the approval link to the remediation deployment. + +## Authentication + +The workload is configured to use OKE Workload Identity first, and falls back to generic OCI resource principals only if workload identity is not available. + +For OKE Workload Identity, the important pieces are: + +- Enhanced OKE cluster. +- Kubernetes `ServiceAccount`. +- IAM policy scoped to: + - `request.principal.type = 'workload'` + - `request.principal.namespace = 'kagent-demo'` + - `request.principal.service_account = 'kagent-oke-remediator'` + - `request.principal.cluster_id = ''` + +No OCI config file mount is required for the manifest included here. + +## Deployment + +Apply: + +```bash +kubectl apply -f k8s/oke-remediator/serviceaccount.yaml +kubectl apply -f k8s/oke-remediator/role.yaml +kubectl apply -f k8s/oke-remediator/rolebinding.yaml +kubectl apply -f k8s/oke-remediator/cronjob.yaml +``` + +Run one job immediately for validation: + +```bash +kubectl create job --from=cronjob/kagent-oke-remediator kagent-oke-remediator-manual -n kagent-demo +kubectl logs job/kagent-oke-remediator-manual -n kagent-demo -f +``` + +## Notes + +- The remediation pipelines must include an approval stage. +- The job triggers the remediation deployment and sends the email with the deployment execution link. +- The job stores the last active incident signature in the `kagent-oke-remediator-state` ConfigMap to suppress duplicate emails for the same active issue. +- The Python worker uses `oci.auth.signers.get_oke_workload_identity_resource_principal_signer()` before any fallback signer. diff --git a/k8s/oke-remediator/cronjob.yaml b/k8s/oke-remediator/cronjob.yaml new file mode 100644 index 0000000..47fa3f7 --- /dev/null +++ b/k8s/oke-remediator/cronjob.yaml @@ -0,0 +1,56 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: kagent-oke-remediator + namespace: kagent-demo +spec: + schedule: "*/3 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 1 + template: + spec: + restartPolicy: Never + serviceAccountName: kagent-oke-remediator + imagePullSecrets: + - name: ocir-secret + containers: + - name: kagent-oke-remediator + image: ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-job:${IMAGE_TAG} + imagePullPolicy: Always + env: + - name: OCI_REGION + value: mx-monterrey-1 + - name: APP_NAMESPACE + value: kagent-demo + - name: KAGENT_MCP_URL + value: http://kagent-controller.kagent.svc.cluster.local:8083/mcp + - name: KAGENT_AGENT_NAME + value: k8s-agent + - name: KAGENT_AGENT_NAMESPACE + value: kagent + - name: ORDER_OBSERVABILITY_URL + value: http://order-service:8080/api/observability + - name: NOTIFICATION_TOPIC_OCID + value: ${NOTIFICATION_TOPIC_OCID} + - name: BUSINESS_REMEDIATION_PIPELINE_URL + value: ${BUSINESS_REMEDIATION_PIPELINE_URL} + - name: BUSINESS_REMEDIATION_PROJECT_OCID + value: ${BUSINESS_REMEDIATION_PROJECT_OCID} + - name: BUSINESS_REMEDIATION_PIPELINE_OCID + value: ${BUSINESS_REMEDIATION_PIPELINE_OCID} + - name: IMAGEPULL_REMEDIATION_PIPELINE_URL + value: ${IMAGEPULL_REMEDIATION_PIPELINE_URL} + - name: IMAGEPULL_REMEDIATION_PROJECT_OCID + value: ${IMAGEPULL_REMEDIATION_PROJECT_OCID} + - name: IMAGEPULL_REMEDIATION_PIPELINE_OCID + value: ${IMAGEPULL_REMEDIATION_PIPELINE_OCID} + - name: HPA_REMEDIATION_PIPELINE_URL + value: ${HPA_REMEDIATION_PIPELINE_URL} + - name: HPA_REMEDIATION_PROJECT_OCID + value: ${HPA_REMEDIATION_PROJECT_OCID} + - name: HPA_REMEDIATION_PIPELINE_OCID + value: ${HPA_REMEDIATION_PIPELINE_OCID} diff --git a/k8s/oke-remediator/role.yaml b/k8s/oke-remediator/role.yaml new file mode 100644 index 0000000..49bf60c --- /dev/null +++ b/k8s/oke-remediator/role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kagent-oke-remediator + namespace: kagent-demo +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update", "patch"] diff --git a/k8s/oke-remediator/rolebinding.yaml b/k8s/oke-remediator/rolebinding.yaml new file mode 100644 index 0000000..f23ec1a --- /dev/null +++ b/k8s/oke-remediator/rolebinding.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kagent-oke-remediator + namespace: kagent-demo +subjects: + - kind: ServiceAccount + name: kagent-oke-remediator + namespace: kagent-demo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kagent-oke-remediator diff --git a/k8s/oke-remediator/serviceaccount.yaml b/k8s/oke-remediator/serviceaccount.yaml new file mode 100644 index 0000000..7b9f30e --- /dev/null +++ b/k8s/oke-remediator/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kagent-oke-remediator + namespace: kagent-demo diff --git a/k8s/remediate-three-errors-command-spec.yaml b/k8s/remediate-three-errors-command-spec.yaml new file mode 100644 index 0000000..ad22b82 --- /dev/null +++ b/k8s/remediate-three-errors-command-spec.yaml @@ -0,0 +1,94 @@ +version: 0.1 +component: command +timeoutInSeconds: 1200 +shell: bash +failImmediatelyOnError: true + +env: + variables: + APP_NAMESPACE: kagent-demo + CONFIGMAP_NAME: demo-config + PAYMENT_DEPLOYMENT: payment-service + PAYMENT_CONTAINER: payment-service + IMAGE_PULL_SECRET: ocir-secret + ORDER_HPA_NAME: order-service + ORDER_MIN_REPLICAS: "3" + ORDER_MAX_REPLICAS: "8" + ORDER_CPU_TARGET: "65" + WAIT_TIMEOUT: 300s + +steps: + - type: Command + name: Fix business ConfigMap + command: | + kubectl patch configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" \ + --type merge \ + -p '{ + "data": { + "ORDER_STUCK_REVIEW_MODE": "false", + "ORDER_MANUAL_REVIEW_RATE_THRESHOLD": "70", + "ORDER_REVENUE_AT_RISK_THRESHOLD": "15000", + "PAYMENT_FORCE_TIMEOUT": "false", + "INVENTORY_FORCE_OUTAGE": "false" + } + }' + + kubectl rollout restart deployment/order-service -n "${APP_NAMESPACE}" + + - type: Command + name: Fix payment image pull secret + command: | + kubectl patch deployment "${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" \ + --type merge \ + -p "{ + \"spec\": { + \"template\": { + \"spec\": { + \"imagePullSecrets\": [ + {\"name\": \"${IMAGE_PULL_SECRET}\"} + ], + \"containers\": [ + { + \"name\": \"${PAYMENT_CONTAINER}\", + \"imagePullPolicy\": \"IfNotPresent\" + } + ] + } + } + } + }" + + kubectl rollout restart deployment/"${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" + + - type: Command + name: Fix order HPA ceiling + command: | + kubectl patch hpa "${ORDER_HPA_NAME}" -n "${APP_NAMESPACE}" \ + --type merge \ + -p "{ + \"spec\": { + \"minReplicas\": ${ORDER_MIN_REPLICAS}, + \"maxReplicas\": ${ORDER_MAX_REPLICAS}, + \"metrics\": [ + { + \"type\": \"Resource\", + \"resource\": { + \"name\": \"cpu\", + \"target\": { + \"type\": \"Utilization\", + \"averageUtilization\": ${ORDER_CPU_TARGET} + } + } + } + ] + } + }" + + - type: Command + name: Wait and verify remediation + command: | + kubectl rollout status deployment/order-service -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}" + kubectl rollout status deployment/"${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}" + kubectl get pods -n "${APP_NAMESPACE}" -o wide + kubectl get hpa -n "${APP_NAMESPACE}" -o wide + kubectl get configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" -o yaml diff --git a/k8s/remediation/README.md b/k8s/remediation/README.md new file mode 100644 index 0000000..48f4dc6 --- /dev/null +++ b/k8s/remediation/README.md @@ -0,0 +1,71 @@ +# Declarative Remediation Manifests + +Use these manifests with OCI DevOps `Apply manifest to your Kubernetes cluster` stages. + +## Business remediation + +Apply in this order: + +```text +business-configmap.yaml +order-rollout-restart.yaml +``` + +`business-configmap.yaml` sets safe business thresholds. + +`order-rollout-restart.yaml` reapplies the `order-service` deployment template so pods reload ConfigMap-backed environment variables. + +Before applying `order-rollout-restart.yaml`, replace: + +```text + +``` + +with a new value, for example the OCI DevOps deployment ID, build number, or timestamp. If the annotation does not change, Kubernetes will not restart the pods. + +## ImagePull remediation + +Use the manifest matching the affected deployment: + +```text +imagepull-all-services.yaml +imagepull-order.yaml +imagepull-payment.yaml +imagepull-inventory.yaml +``` + +For the demo, prefer `imagepull-all-services.yaml`. It restores `ocir-secret` across all three business deployments, so the same remediation pipeline works regardless of which service hit `ImagePullBackOff`. + +Each manifest restores: + +```yaml +imagePullSecrets: + - name: ocir-secret +``` + +Before applying an ImagePull remediation manifest, replace: + +```text + + + + +``` + +with the same values used by the active deployment. The `restart-token` forces a new ReplicaSet after the secret is restored. + +## HPA remediation + +Use: + +```text +order-hpa-capacity.yaml +``` + +It changes `order-service` HPA to: + +```text +minReplicas=3 +maxReplicas=8 +averageUtilization=65 +``` diff --git a/k8s/remediation/business-configmap.yaml b/k8s/remediation/business-configmap.yaml new file mode 100644 index 0000000..de3f373 --- /dev/null +++ b/k8s/remediation/business-configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: demo-config + namespace: kagent-demo +data: + INVENTORY_FORCE_OUTAGE: "false" + PAYMENT_FORCE_TIMEOUT: "false" + PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999" + ORDER_STUCK_REVIEW_MODE: "false" + ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "70" + ORDER_REVENUE_AT_RISK_THRESHOLD: "15000" + INVENTORY_BASE_URL: "http://inventory-service:8081" + PAYMENT_BASE_URL: "http://payment-service:8082" diff --git a/k8s/remediation/imagepull-all-services.yaml b/k8s/remediation/imagepull-all-services.yaml new file mode 100644 index 0000000..d04913b --- /dev/null +++ b/k8s/remediation/imagepull-all-services.yaml @@ -0,0 +1,189 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: order-service + namespace: kagent-demo +spec: + replicas: 1 + selector: + matchLabels: + app: order-service + template: + metadata: + labels: + app: order-service + spec: + imagePullSecrets: + - name: ocir-secret + containers: + - name: order-service + image: .ocir.io//kagent-demo/order-service: + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1024Mi + env: + - name: SERVICES_INVENTORY_BASE_URL + valueFrom: + configMapKeyRef: + name: demo-config + key: INVENTORY_BASE_URL + - name: SERVICES_PAYMENT_BASE_URL + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_BASE_URL + - name: DEMO_ORDERS_STUCK_REVIEW_MODE + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_STUCK_REVIEW_MODE + - name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD + - name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_REVENUE_AT_RISK_THRESHOLD + startupProbe: + httpGet: + path: /actuator/health/liveness + port: 8080 + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payment-service + namespace: kagent-demo +spec: + replicas: 1 + selector: + matchLabels: + app: payment-service + template: + metadata: + labels: + app: payment-service + spec: + imagePullSecrets: + - name: ocir-secret + containers: + - name: payment-service + image: .ocir.io//kagent-demo/payment-service: + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8082 + env: + - name: DEMO_PAYMENT_FORCE_TIMEOUT + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_FORCE_TIMEOUT + - name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX + startupProbe: + httpGet: + path: /actuator/health/liveness + port: 8082 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 18 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8082 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8082 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inventory-service + namespace: kagent-demo +spec: + replicas: 1 + selector: + matchLabels: + app: inventory-service + template: + metadata: + labels: + app: inventory-service + spec: + imagePullSecrets: + - name: ocir-secret + containers: + - name: inventory-service + image: .ocir.io//kagent-demo/inventory-service: + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8081 + env: + - name: DEMO_INVENTORY_FORCE_OUTAGE + valueFrom: + configMapKeyRef: + name: demo-config + key: INVENTORY_FORCE_OUTAGE + startupProbe: + httpGet: + path: /actuator/health/liveness + port: 8081 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 18 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8081 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8081 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 diff --git a/k8s/remediation/imagepull-inventory.yaml b/k8s/remediation/imagepull-inventory.yaml new file mode 100644 index 0000000..1d738d0 --- /dev/null +++ b/k8s/remediation/imagepull-inventory.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inventory-service + namespace: kagent-demo +spec: + replicas: 1 + selector: + matchLabels: + app: inventory-service + template: + metadata: + labels: + app: inventory-service + spec: + imagePullSecrets: + - name: ocir-secret + containers: + - name: inventory-service + image: .ocir.io//kagent-demo/inventory-service: + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8081 + env: + - name: DEMO_INVENTORY_FORCE_OUTAGE + valueFrom: + configMapKeyRef: + name: demo-config + key: INVENTORY_FORCE_OUTAGE + startupProbe: + httpGet: + path: /actuator/health/liveness + port: 8081 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 18 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8081 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8081 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 diff --git a/k8s/remediation/imagepull-order.yaml b/k8s/remediation/imagepull-order.yaml new file mode 100644 index 0000000..f0a8bbe --- /dev/null +++ b/k8s/remediation/imagepull-order.yaml @@ -0,0 +1,76 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: order-service + namespace: kagent-demo +spec: + replicas: 1 + selector: + matchLabels: + app: order-service + template: + metadata: + labels: + app: order-service + spec: + imagePullSecrets: + - name: ocir-secret + containers: + - name: order-service + image: .ocir.io//kagent-demo/order-service: + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1024Mi + env: + - name: SERVICES_INVENTORY_BASE_URL + valueFrom: + configMapKeyRef: + name: demo-config + key: INVENTORY_BASE_URL + - name: SERVICES_PAYMENT_BASE_URL + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_BASE_URL + - name: DEMO_ORDERS_STUCK_REVIEW_MODE + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_STUCK_REVIEW_MODE + - name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD + - name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_REVENUE_AT_RISK_THRESHOLD + startupProbe: + httpGet: + path: /actuator/health/liveness + port: 8080 + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 3 diff --git a/k8s/remediation/imagepull-payment.yaml b/k8s/remediation/imagepull-payment.yaml new file mode 100644 index 0000000..2ebe0a4 --- /dev/null +++ b/k8s/remediation/imagepull-payment.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: payment-service + namespace: kagent-demo +spec: + replicas: 1 + selector: + matchLabels: + app: payment-service + template: + metadata: + labels: + app: payment-service + spec: + imagePullSecrets: + - name: ocir-secret + containers: + - name: payment-service + image: .ocir.io//kagent-demo/payment-service: + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8082 + env: + - name: DEMO_PAYMENT_FORCE_TIMEOUT + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_FORCE_TIMEOUT + - name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX + startupProbe: + httpGet: + path: /actuator/health/liveness + port: 8082 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 18 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8082 + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8082 + initialDelaySeconds: 60 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 diff --git a/k8s/remediation/order-hpa-capacity.yaml b/k8s/remediation/order-hpa-capacity.yaml new file mode 100644 index 0000000..d31205c --- /dev/null +++ b/k8s/remediation/order-hpa-capacity.yaml @@ -0,0 +1,22 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: order-service + namespace: kagent-demo + labels: + app: order-service + demo.oracle.com/remediation: hpa-capacity +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: order-service + minReplicas: 3 + maxReplicas: 8 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 65 diff --git a/k8s/remediation/order-rollout-restart.yaml b/k8s/remediation/order-rollout-restart.yaml new file mode 100644 index 0000000..52e77ed --- /dev/null +++ b/k8s/remediation/order-rollout-restart.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: order-service + namespace: kagent-demo +spec: + replicas: 1 + selector: + matchLabels: + app: order-service + template: + metadata: + labels: + app: order-service + annotations: + demo.oracle.com/restart-token: "" + spec: + imagePullSecrets: + - name: ocir-secret + containers: + - name: order-service + image: .ocir.io//kagent-demo/order-service:latest + ports: + - containerPort: 8080 + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1024Mi + env: + - name: SERVICES_INVENTORY_BASE_URL + valueFrom: + configMapKeyRef: + name: demo-config + key: INVENTORY_BASE_URL + - name: SERVICES_PAYMENT_BASE_URL + valueFrom: + configMapKeyRef: + name: demo-config + key: PAYMENT_BASE_URL + - name: DEMO_ORDERS_STUCK_REVIEW_MODE + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_STUCK_REVIEW_MODE + - name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD + - name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD + valueFrom: + configMapKeyRef: + name: demo-config + key: ORDER_REVENUE_AT_RISK_THRESHOLD + startupProbe: + httpGet: + path: /actuator/health/liveness + port: 8080 + failureThreshold: 30 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /actuator/health/readiness + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /actuator/health/liveness + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 3 diff --git a/oke-remediator-multi-check/main.py b/oke-remediator-multi-check/main.py new file mode 100644 index 0000000..2b89cb5 --- /dev/null +++ b/oke-remediator-multi-check/main.py @@ -0,0 +1,1341 @@ +"""Periodic OKE remediator that runs three independent kagent checks. + +The job performs: +1. Business application health validation for Orders. +2. HPA health validation for order-service. +3. ImagePull health validation for application deployments. + +Each check is sent to kagent as an independent MCP task. The returned JSON is +normalized into a stable machine-readable shape so the job can: +- decide whether the issue is active, +- avoid duplicate notifications for the same active incident, +- trigger the matching OCI DevOps remediation deployment, +- send one email report with the approval link for that remediation. +""" + +import asyncio +import ast +import hashlib +import json +import os +import re +import sys +import traceback +import urllib.request +import uuid +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List +from urllib.error import HTTPError + +import oci +from mcp import ClientSession + +try: + from mcp.client.streamable_http import streamable_http_client as streamablehttp_client +except ImportError: + from mcp.client.streamable_http import streamablehttp_client + + +STATE_CONFIGMAP_NAME = "kagent-oke-remediator-multi-state" +WORKLOAD_NAME = "kagent-oke-remediator-multi" +HEALTHY_STATUSES = { + "HEALTHY", + "OK", + "COMPLETED", + "ALL SYSTEMS OPERATIONAL", + "NO ACTIVE AUTOSCALING", + "NO ACTIVE IMAGE_PULL_FAILURE DETECTED", + "NO FAILURE", + "NO ISSUE DETECTED", + "NO HAY ERRORES DE OBTENCIÓN DE IMÁGENES.", + "NO HAY ERRORES DE OBTENCION DE IMAGENES.", +} +IMAGE_PULL_WAITING_REASONS = { + "ERRIMAGEPULL", + "IMAGEPULLBACKOFF", + "INVALIDIMAGENAME", + "CREATECONTAINERCONFIGERROR", +} +IMAGE_PULL_EVENT_REASONS = { + "FAILED", + "BACKOFF", + "INSPECTFAILED", +} +INTERNAL_FAILURE_TYPES = { + "AGENT_RESPONSE_PARSE_FAILURE", + "AGENT_INPUT_REQUIRED", + "JOB_RUNTIME_FAILURE", +} + + +# Structured logging is used heavily because this job is executed as a CronJob. +# The easiest way to understand behavior later is by reading pod logs, so every +# major phase emits a compact JSON event. +def _log_event(event: str, fields: Dict[str, Any]) -> None: + """Emit a single structured log event for easy scraping from pod logs.""" + print(json.dumps({"event": event, **fields}, ensure_ascii=False), flush=True) + + +def main() -> int: + """Entrypoint used by the container process.""" + app_namespace = os.environ.get("APP_NAMESPACE", "kagent-demo") + region = os.environ.get("OCI_REGION", "mx-monterrey-1") + kagent_url = os.environ.get("KAGENT_MCP_URL", "") + _log_event( + "KAGENT_MULTI_START", + {"namespace": app_namespace, "region": region, "kagentMcpUrl": kagent_url}, + ) + try: + result = asyncio.run(run_multi_checks()) + _log_event( + "KAGENT_MULTI_RESULT", + { + "namespace": app_namespace, + "resultsCount": len(result.get("results", [])), + "notificationsSent": len(result.get("notificationsSent", [])), + "remediationsTriggered": len(result.get("remediationsTriggered", [])), + }, + ) + print(json.dumps(result, indent=2, ensure_ascii=False), flush=True) + return 0 + except Exception as exc: + error = { + "status": "FAILED", + "failureType": "JOB_RUNTIME_FAILURE", + "rootCause": _exception_summary(exc), + "traceback": "".join(traceback.format_exception(exc)), + } + print(json.dumps(error, indent=2, ensure_ascii=False), flush=True) + return 1 + + +def _fetch_json(url: str) -> Dict[str, Any]: + """Fetch the Orders observability payload used by the business check.""" + if not url: + return {"fetchStatus": "SKIPPED", "reason": "observabilityUrl is empty"} + + try: + with urllib.request.urlopen(url, timeout=10) as response: + body = response.read(20000).decode("utf-8") + return json.loads(body) + except Exception as exc: + return { + "fetchStatus": "FAILED", + "error": _exception_summary(exc), + } + + +async def run_multi_checks() -> Dict[str, Any]: + """Run the three checks, deduplicate incidents, trigger remediation, send mail.""" + payload = _build_payload_from_env() + results: List[Dict[str, Any]] = [] + notifications_sent: List[str] = [] + remediations_triggered: List[Dict[str, Any]] = [] + + # Each prompt is executed as an independent MCP interaction so kagent treats + # them like separate conversations/checks instead of sharing one context. + for check in _build_checks(payload): + result = await _run_single_check(payload, check) + results.append(result) + + # Persist one state entry per error_code so a still-open incident does not + # create another remediation deployment or another email every 3 minutes. + state = _load_notification_state(payload["appNamespace"]) + for result in results: + result = _normalize_result(result, payload["appNamespace"]) + error_code = result.get("error_code", "") + + # When a check returns healthy again we clear only that incident family. + # This allows the same issue to be detected as "new" in the future. + if not _is_active_failure(result): + _clear_state_for_code(payload["appNamespace"], state, error_code) + continue + + if _is_internal_failure(result): + _log_event( + "KAGENT_MULTI_INTERNAL_FAILURE", + { + "check": result.get("checkName", ""), + "errorCode": error_code, + "failureType": result.get("failureType", ""), + "rootCause": result.get("rootCause", ""), + }, + ) + _clear_state_for_code(payload["appNamespace"], state, error_code) + result["emailSent"] = False + result["notificationReason"] = "Internal agent/runtime failure; notification suppressed." + _replace_result(results, result) + continue + + signature = _build_signature(result) + state_entry = state.get(error_code, {}) + active_url = state_entry.get("deploymentUrl", "") + active_signature = state_entry.get("signature", "") + active_email = state_entry.get("emailSent", "") + + # If the same incident is still active and we already have a remediation + # deployment link for it, suppress new emails and new deployments. + if signature == active_signature and active_url and active_email == "true": + result["emailSent"] = False + result["notificationReason"] = "Duplicate active issue; notification suppressed." + result["remediationDeployment"] = { + "triggered": True, + "deploymentUrl": active_url, + "deploymentId": state_entry.get("deploymentId", ""), + "reused": True, + } + continue + + remediation = _safe_trigger_remediation(result, payload) + if remediation: + result["remediationDeployment"] = remediation + if remediation.get("triggered"): + remediations_triggered.append(remediation) + + email_sent, notification_reason = _safe_send_email(result, payload) + result["emailSent"] = email_sent + result["notificationReason"] = notification_reason + if email_sent: + notifications_sent.append(error_code) + + # Persist state only after this check has finished its notification/ + # remediation cycle so future runs know whether the incident is new. + if email_sent and (result.get("remediationDeployment") or {}).get("deploymentUrl"): + state[error_code] = { + "signature": signature, + "deploymentUrl": result["remediationDeployment"].get("deploymentUrl", ""), + "deploymentId": result["remediationDeployment"].get("deploymentId", ""), + "emailSent": "true", + } + else: + state[error_code] = { + "signature": "", + "deploymentUrl": "", + "deploymentId": "", + "emailSent": "false", + } + + _save_notification_state(payload["appNamespace"], state) + return { + "status": "COMPLETED", + "results": results, + "notificationsSent": notifications_sent, + "remediationsTriggered": remediations_triggered, + } + + +def _replace_result(results: List[Dict[str, Any]], updated: Dict[str, Any]) -> None: + """Replace the stored result for one check without changing list order.""" + for idx, item in enumerate(results): + if item.get("checkName") == updated.get("checkName"): + results[idx] = updated + return + + +def _build_payload_from_env() -> Dict[str, Any]: + """Collect all runtime settings from environment variables.""" + return { + "region": os.environ.get("OCI_REGION", "mx-monterrey-1"), + "appNamespace": os.environ.get("APP_NAMESPACE", "kagent-demo"), + "kagentMcpUrl": os.environ.get("KAGENT_MCP_URL", ""), + "agentName": os.environ.get("KAGENT_AGENT_NAME", "k8s-agent"), + "agentNamespace": os.environ.get("KAGENT_AGENT_NAMESPACE", "kagent"), + "observabilityUrl": os.environ.get("ORDER_OBSERVABILITY_URL", ""), + "notificationTopicOcid": os.environ.get("NOTIFICATION_TOPIC_OCID", ""), + "remediationPipelines": { + "business": { + "projectOcid": os.environ.get("BUSINESS_REMEDIATION_PROJECT_OCID", ""), + "pipelineOcid": os.environ.get("BUSINESS_REMEDIATION_PIPELINE_OCID", ""), + }, + "imagePull": { + "projectOcid": os.environ.get("IMAGEPULL_REMEDIATION_PROJECT_OCID", ""), + "pipelineOcid": os.environ.get("IMAGEPULL_REMEDIATION_PIPELINE_OCID", ""), + }, + "hpa": { + "projectOcid": os.environ.get("HPA_REMEDIATION_PROJECT_OCID", ""), + "pipelineOcid": os.environ.get("HPA_REMEDIATION_PIPELINE_OCID", ""), + }, + }, + } + + +async def _run_single_check(payload: Dict[str, Any], check: Dict[str, str]) -> Dict[str, Any]: + """Open a fresh MCP interaction for one check and return the parsed payload.""" + context_id = f"{check['name']}-{uuid.uuid4()}" + _log_event("KAGENT_MULTI_CHECK_START", {"check": check["name"], "contextId": context_id}) + async with streamablehttp_client(payload["kagentMcpUrl"]) as streams: + read_stream, write_stream = streams[:2] + async with ClientSession(read_stream, write_stream) as session: + await session.initialize() + response = await session.call_tool( + "invoke_agent", + { + "agent": f"{payload['agentNamespace']}/{payload['agentName']}", + "task": check["prompt"], + "context_id": context_id, + }, + ) + + text = _extract_text(response) + result = _parse_json_response(text) + result["checkName"] = check["name"] + result.setdefault("rawResponse", text) + _log_event( + "KAGENT_MULTI_CHECK_RESULT", + { + "check": check["name"], + "status": result.get("status", ""), + "failureType": result.get("failureType", ""), + "errorCode": result.get("error_code", ""), + }, + ) + return result + + +def _extract_text(response: Any) -> str: + """Flatten MCP tool response content into one text block.""" + parts = [] + for item in getattr(response, "content", []) or []: + text = getattr(item, "text", None) + if text: + parts.append(text) + return "\n".join(parts) if parts else str(response) + + +def _parse_json_response(text: str) -> Dict[str, Any]: + """Extract and parse the first JSON object returned by kagent.""" + candidate = _extract_json_candidate(text) + if not candidate: + return { + "status": "FAILED", + "failureType": "AGENT_RESPONSE_PARSE_FAILURE", + "rootCause": "kagent did not return JSON", + "rawResponse": text, + } + try: + return json.loads(candidate) + except json.JSONDecodeError: + parsed = ast.literal_eval(candidate) + if not isinstance(parsed, dict): + raise ValueError("Parsed agent payload is not a JSON object") + return parsed + + +def _extract_json_candidate(text: str) -> str: + """Prefer fenced ```json blocks; otherwise take the outermost object.""" + fenced_match = re.search(r"```json\s*(\{.*?\})\s*```", text, re.DOTALL | re.IGNORECASE) + if fenced_match: + return fenced_match.group(1) + start = text.find("{") + end = text.rfind("}") + if start == -1 or end == -1 or end <= start: + return "" + return text[start : end + 1] + + +def _normalize_result(result: Dict[str, Any], app_namespace: str) -> Dict[str, Any]: + """Normalize the heterogeneous kagent responses into a stable contract.""" + normalized = dict(result) + normalized.setdefault("namespace", app_namespace) + + # First normalize the "incident family" so downstream logic knows which + # remediation pipeline belongs to this result. + normalized["error_code"] = _normalize_error_code(normalized) + + # Then normalize health/failure semantics so prose like "No Failure" or + # "No issue detected" is converted to a machine-usable state. + normalized["status"] = _normalize_status(normalized) + normalized["failureType"] = _normalize_failure_type(normalized) + + # The report is what goes to the email body. We flatten dict-shaped evidence + # into a list so the email renderer can print one bullet per line. + if isinstance(normalized.get("report"), dict): + report = dict(normalized["report"]) + if isinstance(report.get("evidence"), dict): + report["evidence"] = [f"{key}: {value}" for key, value in report["evidence"].items()] + normalized["report"] = report + if normalized["error_code"] == "BUSINESS_APPLICATION" and "pending_transactions" not in normalized: + normalized["pending_transactions"] = _extract_pending_transactions(normalized) + if normalized["error_code"] == "BUSINESS_APPLICATION": + normalized = _reconcile_business_with_observability(normalized) + if normalized["error_code"] == "IMAGE_PULL": + normalized = _reconcile_imagepull_with_live_state(normalized, app_namespace) + return normalized + + +def _reconcile_business_with_observability(result: Dict[str, Any]) -> Dict[str, Any]: + """Suppress business hallucinations when observability clearly says the business flow is healthy.""" + observability_url = os.environ.get("ORDER_OBSERVABILITY_URL", "") + observability_json = _compact_observability_json(_fetch_json(observability_url)) + business = observability_json.get("businessHealth", {}) + services = observability_json.get("services", []) + + healthy = bool(business.get("healthy", True)) + total_orders = _to_number(business.get("totalOrders", 0)) + pending_orders = _to_number(business.get("pendingReviewOrders", 0)) + failed_orders = _to_number(business.get("failedOrders", 0)) + manual_review_rate = _to_number(business.get("manualReviewRate", 0)) + revenue_at_risk = _to_number(business.get("revenueAtRisk", 0)) + degraded_services = [ + service.get("name", "") + for service in services + if isinstance(service, dict) + and str(service.get("status", "")).strip().upper() == "DEGRADED" + and bool(service.get("kagentSignal", False)) + ] + + if healthy and total_orders == 0 and pending_orders == 0 and failed_orders == 0 and manual_review_rate == 0 and revenue_at_risk == 0 and not degraded_services: + normalized = dict(result) + normalized["status"] = "HEALTHY" + normalized["failureType"] = "NO_ACTIVE_FAILURE" + normalized["pending_transactions"] = 0 + normalized["summary"] = "No hay degradación activa del proceso de negocio según la observabilidad actual." + normalized["rootCause"] = "La observabilidad actual muestra flujo de negocio saludable y sin órdenes pendientes, fallidas o revenue en riesgo." + normalized["businessImpact"] = "" + normalized["recommendedAction"] = "No se requiere remediación de negocio. Continúe monitoreando señales de runtime y negocio." + normalized["evidence"] = [ + f"businessHealth.healthy={healthy}", + f"totalOrders={total_orders}", + f"pendingReviewOrders={pending_orders}", + f"failedOrders={failed_orders}", + f"manualReviewRate={manual_review_rate}", + f"revenueAtRisk={revenue_at_risk}", + "degradedBusinessServices=none", + ] + normalized["report"] = { + "executiveSummary": "La observabilidad actual confirma que no existe degradación activa del proceso de negocio.", + "rootCause": normalized["rootCause"], + "businessImpact": "", + "recommendedAction": normalized["recommendedAction"], + "evidence": list(normalized["evidence"]), + "recommendedConfigMapData": {}, + } + return normalized + + return result + + +def _to_number(value: Any) -> float: + try: + return float(value) + except (TypeError, ValueError): + return 0.0 + + +def _reconcile_imagepull_with_live_state(result: Dict[str, Any], namespace: str) -> Dict[str, Any]: + """Ignore historical image-pull incidents when no current pod has an active pull error.""" + snapshot = _get_live_imagepull_snapshot(namespace) + if snapshot["active"]: + if _is_model_runtime_failure(result): + return _build_live_imagepull_result(result, snapshot, namespace) + evidence = result.get("evidence", []) + if not isinstance(evidence, list): + evidence = [str(evidence)] if evidence else [] + for line in snapshot.get("evidence", []): + if line not in evidence: + evidence.append(line) + result["evidence"] = evidence + return result + + normalized = dict(result) + normalized["status"] = "HEALTHY" + normalized["failureType"] = "NO_ACTIVE_FAILURE" + normalized["rootCause"] = snapshot["reason"] + normalized["businessImpact"] = "" + evidence = normalized.get("evidence", []) + if not isinstance(evidence, list): + evidence = [str(evidence)] if evidence else [] + for line in snapshot.get("evidence", []): + if line not in evidence: + evidence.append(line) + normalized["evidence"] = evidence + report = normalized.get("report", {}) + if isinstance(report, dict): + report = dict(report) + report["rootCause"] = normalized["rootCause"] + report["businessImpact"] = "" + report["recommendedAction"] = "No remediation required for image pull. Current pods are not failing due to image extraction." + report_evidence = report.get("evidence", []) + if not isinstance(report_evidence, list): + report_evidence = [str(report_evidence)] if report_evidence else [] + for line in snapshot.get("evidence", []): + if line not in report_evidence: + report_evidence.append(line) + report["evidence"] = report_evidence + normalized["report"] = report + normalized["recommendedAction"] = "No remediation required for image pull. Current pods are not failing due to image extraction." + return normalized + + +def _build_live_imagepull_result(result: Dict[str, Any], snapshot: Dict[str, Any], namespace: str) -> Dict[str, Any]: + """Build a real IMAGE_PULL incident from live cluster evidence when the model response failed.""" + active_pod = snapshot.get("activePod", "") + affected_deployment = _deployment_name_from_pod(active_pod) + affected_image = snapshot.get("affectedImage", "") + evidence = list(snapshot.get("evidence", [])) + + normalized = dict(result) + normalized["error_code"] = "IMAGE_PULL" + normalized["status"] = "FAILED" + normalized["failureType"] = "IMAGE_PULL_FAILURE" + normalized["affectedDeployment"] = affected_deployment + normalized["affectedPod"] = active_pod + normalized["affectedImage"] = affected_image + normalized["imagePullSecret"] = normalized.get("imagePullSecret", "") + normalized["oldReplicasAvailable"] = normalized.get("oldReplicasAvailable", 0) + normalized["newReplicaSetReadyReplicas"] = normalized.get("newReplicaSetReadyReplicas", 0) + normalized["rootCause"] = "Current pod state confirms an active image pull failure; model analysis was unavailable due to context length limits." + normalized["businessImpact"] = "The affected workload cannot start correctly, which can degrade application availability and delay business processing." + normalized["evidence"] = evidence + normalized["recommendedAction"] = "Review the image reference and registry credentials, then rerun the remediation pipeline for image pull recovery." + normalized["recommendedImagePullFix"] = { + "deploymentName": affected_deployment, + "secretName": normalized.get("imagePullSecret", ""), + "image": affected_image, + } + normalized["report"] = { + "executiveSummary": "Se confirmó una falla activa de image pull a partir del estado actual del pod en Kubernetes, por lo que el incidente debe tratarse como real aunque el análisis del modelo no haya completado.", + "rootCause": normalized["rootCause"], + "businessImpact": normalized["businessImpact"], + "recommendedAction": normalized["recommendedAction"], + "evidence": evidence, + } + _log_event( + "IMAGE_PULL_LIVE_FALLBACK", + { + "namespace": namespace, + "affectedPod": active_pod, + "affectedDeployment": affected_deployment, + "affectedImage": affected_image, + }, + ) + return normalized + + +def _get_live_imagepull_snapshot(namespace: str) -> Dict[str, Any]: + """Return the current namespace image-pull state, prioritizing live pod state over history.""" + try: + pods_response = _kube_api_request("GET", f"/api/v1/namespaces/{namespace}/pods") + except Exception as exc: + _log_event( + "IMAGE_PULL_LIVE_CHECK_FAILED", + {"namespace": namespace, "stage": "pods", "error": _exception_summary(exc)}, + ) + return { + "active": False, + "reason": "Live pod verification failed, so historical image pull events were ignored to avoid false positives.", + "evidence": [f"livePodImagePullCheckError={_exception_summary(exc)}"], + } + + active_pods = [] + pod_evidence: List[str] = [] + affected_image = "" + for item in pods_response.get("items", []): + pod_name = item.get("metadata", {}).get("name", "") + status = item.get("status", {}) + for container_status in status.get("containerStatuses", []) or []: + reason = _container_waiting_reason(container_status) + if reason in IMAGE_PULL_WAITING_REASONS: + active_pods.append(pod_name) + pod_evidence.append(f"activeImagePullPod={pod_name} reason={reason}") + if not affected_image: + affected_image = str(container_status.get("image", "")).strip() + for container_status in status.get("initContainerStatuses", []) or []: + reason = _container_waiting_reason(container_status) + if reason in IMAGE_PULL_WAITING_REASONS: + active_pods.append(pod_name) + pod_evidence.append(f"activeImagePullInitPod={pod_name} reason={reason}") + if not affected_image: + affected_image = str(container_status.get("image", "")).strip() + + if active_pods: + recent_event_evidence = _recent_imagepull_event_evidence(namespace, active_pods) + return { + "active": True, + "reason": "Current pod state shows an active image pull failure.", + "activePod": active_pods[0], + "affectedImage": affected_image, + "evidence": pod_evidence + recent_event_evidence, + } + + return { + "active": False, + "reason": "No current pod in the namespace is showing an active image pull error.", + "evidence": ["livePodImagePullState=clean", "livePodPhaseSummary=all-current-pods-running-or-completed"], + } + + +def _container_has_imagepull_reason(container_status: Dict[str, Any]) -> bool: + """Check current waiting state for image-pull-specific failure reasons.""" + reason = _container_waiting_reason(container_status) + return reason in IMAGE_PULL_WAITING_REASONS + + +def _container_waiting_reason(container_status: Dict[str, Any]) -> str: + waiting = ((container_status or {}).get("state") or {}).get("waiting") or {} + return str(waiting.get("reason", "")).strip().upper() + + +def _recent_imagepull_event_evidence(namespace: str, pod_names: List[str], lookback_minutes: int = 15) -> List[str]: + """Capture only recent image-pull-related events for currently failing pods.""" + try: + events_response = _kube_api_request("GET", f"/api/v1/namespaces/{namespace}/events") + except Exception as exc: + _log_event( + "IMAGE_PULL_LIVE_CHECK_FAILED", + {"namespace": namespace, "stage": "events", "error": _exception_summary(exc)}, + ) + return [f"recentImagePullEventCheckError={_exception_summary(exc)}"] + + threshold = datetime.now(timezone.utc) - timedelta(minutes=lookback_minutes) + pod_name_set = {name for name in pod_names if name} + evidence: List[str] = [] + for item in events_response.get("items", []): + involved_name = item.get("involvedObject", {}).get("name", "") + if involved_name not in pod_name_set: + continue + reason = str(item.get("reason", "")).strip().upper() + message = str(item.get("message", "")).strip() + if reason not in IMAGE_PULL_EVENT_REASONS and not any( + token in message.upper() for token in IMAGE_PULL_WAITING_REASONS + ): + continue + event_time = _parse_k8s_event_time(item) + if event_time and event_time < threshold: + continue + evidence.append(f"recentEvent pod={involved_name} reason={reason} message={message[:180]}") + return evidence + + +def _parse_k8s_event_time(item: Dict[str, Any]) -> datetime | None: + """Parse the most useful event timestamp field when present.""" + for key in ("eventTime", "lastTimestamp", "firstTimestamp", "metadata.creationTimestamp"): + value = item.get(key) + if value is None and "." in key: + first, second = key.split(".", 1) + value = ((item.get(first) or {}).get(second)) + if not value: + continue + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + continue + return None + + +def _deployment_name_from_pod(pod_name: str) -> str: + """Best-effort extraction of deployment name from a ReplicaSet-style pod name.""" + if not pod_name: + return "" + parts = pod_name.split("-") + if len(parts) >= 3: + return "-".join(parts[:-2]) + return pod_name + + +def _normalize_error_code(result: Dict[str, Any]) -> str: + """Map free-form responses back to one of the three supported error codes.""" + raw = str(result.get("error_code", "")).strip().upper() + check_name = str(result.get("checkName", "")).strip().lower() + failure_type = str(result.get("failureType", "")).strip().upper() + status = str(result.get("status", "")).strip().upper() + problems = result.get("problems", []) + markers = set() + if failure_type: + markers.add(failure_type) + if isinstance(problems, list): + for problem in problems: + if isinstance(problem, str): + markers.add(problem.upper()) + elif isinstance(problem, dict): + if problem.get("failureType"): + markers.add(str(problem.get("failureType")).upper()) + if problem.get("type"): + markers.add(str(problem.get("type")).upper()) + + if raw in {"BUSINESS_APPLICATION", "HPA", "IMAGE_PULL"}: + return raw + if "IMAGE_PULL_FAILURE" in markers: + return "IMAGE_PULL" + if "HPA_AUTOSCALING_FAILURE" in markers: + return "HPA" + if raw in {"NO_ERROR", "NONE"} or status in { + "NO FAILURE", + "NO ISSUE DETECTED", + "NO HAY ERRORES DE OBTENCIÓN DE IMÁGENES.", + "NO HAY ERRORES DE OBTENCION DE IMAGENES.", + }: + if check_name == "hpa-failures": + return "HPA" + if check_name == "imagepull-error": + return "IMAGE_PULL" + if check_name == "hpa-failures": + return "HPA" + if check_name == "imagepull-error": + return "IMAGE_PULL" + return "BUSINESS_APPLICATION" + + +def _normalize_status(result: Dict[str, Any]) -> str: + """Collapse non-standard healthy phrases to HEALTHY and active issues to DEGRADED.""" + raw = str(result.get("status", "")).strip().upper() + failure_type = str(result.get("failureType", "") or "").strip().upper() + if raw in HEALTHY_STATUSES: + return "HEALTHY" + if raw in {"HEALTHY", "DEGRADED", "FAILED"}: + return raw + if failure_type in {"NONE", "NO_ACTIVE_FAILURE", "NO_HPA_DEFINED"}: + return "HEALTHY" + if failure_type in {"HPA_AUTOSCALING_FAILURE", "IMAGE_PULL_FAILURE", "BUSINESS_PROCESS_DEGRADATION"}: + return "DEGRADED" + return raw or "FAILED" + + +def _normalize_failure_type(result: Dict[str, Any]) -> str: + """Infer a canonical failure type when kagent omits or varies the field.""" + failure_type = str(result.get("failureType", "") or "").strip().upper() + status = str(result.get("status", "")).strip().upper() + if _is_model_runtime_failure(result): + return "JOB_RUNTIME_FAILURE" + if failure_type in {"NONE", "NO_HPA_DEFINED"}: + return "NO_ACTIVE_FAILURE" + if result.get("error_code") in {"HPA", "IMAGE_PULL"} and status in HEALTHY_STATUSES: + return "NO_ACTIVE_FAILURE" + if failure_type: + return failure_type + + problems = result.get("problems", []) + if isinstance(problems, list): + for problem in problems: + if isinstance(problem, str) and problem.upper() in {"IMAGE_PULL_FAILURE", "HPA_AUTOSCALING_FAILURE", "BUSINESS_PROCESS_DEGRADATION"}: + return problem.upper() + if isinstance(problem, dict): + if problem.get("failureType"): + return str(problem.get("failureType")).upper() + if problem.get("type"): + return str(problem.get("type")).upper() + + if result.get("error_code") in {"HPA", "IMAGE_PULL"} and _normalize_status(result) == "HEALTHY": + return "NO_ACTIVE_FAILURE" + return "" + + +def _is_model_runtime_failure(result: Dict[str, Any]) -> bool: + """Detect provider/model failures so they are suppressed instead of remediated.""" + code = str(result.get("code", "")).strip() + message = str(result.get("message", "")).strip().lower() + raw_response = str(result.get("rawResponse", "")).strip().lower() + root_cause = str(result.get("rootCause", "")).strip().lower() + payload = " ".join(part for part in [message, raw_response, root_cause] if part) + + if code == "400" and "context_length_exceeded" in payload: + return True + if "context_length_exceeded" in payload: + return True + if "invalid_request_error" in payload and "maximum context length" in payload: + return True + return False + + +def _extract_pending_transactions(result: Dict[str, Any]) -> int: + """Best-effort extraction of pending transactions from evidence lines.""" + evidence = result.get("evidence", []) + if isinstance(evidence, list): + for item in evidence: + text = str(item) + if text.startswith("pendingReviewOrders="): + try: + return int(float(text.split("=", 1)[1])) + except ValueError: + return 0 + return 0 + + +def _compact_observability_json(data: Dict[str, Any]) -> Dict[str, Any]: + """Keep the business prompt stateless but bounded so it does not grow with runtime history.""" + if not isinstance(data, dict): + return {"fetchStatus": "FAILED", "error": "Observability payload is not a JSON object"} + + business = data.get("businessHealth", {}) if isinstance(data.get("businessHealth"), dict) else {} + services = data.get("services", []) if isinstance(data.get("services"), list) else [] + orders = data.get("recentOrders", []) if isinstance(data.get("recentOrders"), list) else [] + recommendations = data.get("recommendedKagentActions", []) + if not isinstance(recommendations, list): + recommendations = [str(recommendations)] if recommendations else [] + + compact_orders = [] + for order in orders[:5]: + if not isinstance(order, dict): + continue + compact_orders.append( + { + "orderId": order.get("orderId", ""), + "status": order.get("status", ""), + "amount": order.get("amount", 0), + "reason": order.get("reason", ""), + "createdAt": order.get("createdAt", ""), + } + ) + + compact_services = [] + for service in services[:6]: + if not isinstance(service, dict): + continue + compact_services.append( + { + "name": service.get("name", ""), + "layer": service.get("layer", ""), + "status": service.get("status", ""), + "detail": service.get("detail", ""), + "kagentSignal": service.get("kagentSignal", False), + } + ) + + return { + "timestamp": data.get("timestamp", ""), + "businessHealth": { + "totalOrders": business.get("totalOrders", 0), + "confirmedOrders": business.get("confirmedOrders", 0), + "pendingReviewOrders": business.get("pendingReviewOrders", 0), + "failedOrders": business.get("failedOrders", 0), + "manualReviewRate": business.get("manualReviewRate", 0), + "totalRevenue": business.get("totalRevenue", 0), + "confirmedRevenue": business.get("confirmedRevenue", 0), + "revenueAtRisk": business.get("revenueAtRisk", 0), + "healthy": business.get("healthy", True), + "summary": business.get("summary", ""), + "businessImpact": business.get("businessImpact", ""), + }, + "services": compact_services, + "recentOrders": compact_orders, + "recommendedKagentActions": recommendations[:5], + } + + +def _trigger_remediation(result: Dict[str, Any], payload: Dict[str, Any]) -> Dict[str, Any]: + """Create the remediation deployment in OCI DevOps for one incident type.""" + pipeline_key = { + "BUSINESS_APPLICATION": "business", + "IMAGE_PULL": "imagePull", + "HPA": "hpa", + }.get(result.get("error_code", ""), "business") + pipeline = payload["remediationPipelines"].get(pipeline_key, {}) + pipeline_ocid = pipeline.get("pipelineOcid", "") + project_ocid = pipeline.get("projectOcid", "") + if not pipeline_ocid: + return {"triggered": False, "reason": "No remediation pipeline OCID was provided."} + + client = oci.devops.DevopsClient( + config={"region": payload["region"]}, + signer=oci.auth.signers.get_oke_workload_identity_resource_principal_signer(), + ) + response = client.create_deployment( + create_deployment_details=oci.devops.models.CreateDeployPipelineDeploymentDetails( + deploy_pipeline_id=pipeline_ocid, + display_name=f"oke-remediator-{result.get('error_code', 'UNKNOWN').lower()}-{uuid.uuid4().hex[:8]}", + ), + opc_retry_token=str(uuid.uuid4()), + ) + deployment_id = response.data.id + return { + "triggered": True, + "deploymentId": deployment_id, + "deploymentUrl": ( + f"https://cloud.oracle.com/devops-deployment/projects/{project_ocid}/pipelines/" + f"{pipeline_ocid}/deployments/{deployment_id}?region={payload['region']}" + ), + } + + +def _safe_trigger_remediation(result: Dict[str, Any], payload: Dict[str, Any]) -> Dict[str, Any]: + """Return a non-throwing remediation result so one failing check does not abort the whole job.""" + try: + return _trigger_remediation(result, payload) + except Exception as exc: + return {"triggered": False, "reason": f"Failed to trigger remediation deployment: {_exception_summary(exc)}"} + + +def _maybe_send_email(result: Dict[str, Any], payload: Dict[str, Any]) -> tuple[bool, str]: + """Publish one OCI Notifications email for an actionable incident.""" + topic_id = payload.get("notificationTopicOcid", "") + if not topic_id: + return False, "NOTIFICATION_TOPIC_OCID is not configured." + + client = oci.ons.NotificationDataPlaneClient( + config={"region": payload["region"]}, + signer=oci.auth.signers.get_oke_workload_identity_resource_principal_signer(), + ) + client.publish_message( + topic_id=topic_id, + message_details=oci.ons.models.MessageDetails( + title=_subject_for_result(result), + body=_body_for_result(result, payload["region"]), + ), + ) + return True, "Failure report published to OCI Notifications." + + +def _safe_send_email(result: Dict[str, Any], payload: Dict[str, Any]) -> tuple[bool, str]: + """Return a non-throwing notification result for the current incident.""" + try: + return _maybe_send_email(result, payload) + except Exception as exc: + return False, f"Failed to publish notification: {_exception_summary(exc)}" + + +def _subject_for_result(result: Dict[str, Any]) -> str: + """Build a stable subject based on namespace, status and error code.""" + return f"[OKE Job][kagent][{result.get('namespace', '')}] {result.get('status', '')} - {result.get('error_code', '')}"[:255] + + +def _body_for_result(result: Dict[str, Any], region: str) -> str: + """Render the email body as a compact operational report.""" + report = result.get("report", {}) if isinstance(result.get("report"), dict) else {} + remediation = result.get("remediationDeployment", {}) if isinstance(result.get("remediationDeployment"), dict) else {} + deployment_url = remediation.get("deploymentUrl", "") + evidence = report.get("evidence", result.get("evidence", [])) + if not isinstance(evidence, list): + evidence = [str(evidence)] + + lines = [ + "kagent diagnostic report", + "", + f"Region: {region}", + f"Namespace: {result.get('namespace', '')}", + f"Status: {result.get('status', '')}", + f"Error code: {result.get('error_code', '')}", + f"Failure type: {result.get('failureType', '')}", + "", + "Executive summary:", + str(report.get("executiveSummary", result.get("summary", ""))), + "", + "Root cause:", + str(report.get("rootCause", result.get("rootCause", ""))), + "", + "Business impact:", + str(report.get("businessImpact", result.get("businessImpact", ""))), + "", + "Evidence:", + ] + # Keep the body compact enough for email while still including the most + # useful evidence lines from the check. + lines.extend(f"- {item}" for item in evidence[:12]) + lines.extend( + [ + "", + "Recommended action:", + str(report.get("recommendedAction", result.get("recommendedAction", ""))), + "", + "Remediation deployment:", + f"- Triggered: {remediation.get('triggered', False)}", + f"- Deployment ID: {remediation.get('deploymentId', '')}", + ] + ) + if deployment_url: + lines.extend(["", "Approve or reject remediation deployment in OCI DevOps:", deployment_url]) + elif remediation.get("reason"): + lines.extend(["", "Remediation deployment was not created:", str(remediation.get("reason", ""))]) + return "\n".join(lines) + + +def _build_signature(result: Dict[str, Any]) -> str: + """Build a stable signature used to suppress duplicate active incidents.""" + payload = { + "namespace": result.get("namespace", ""), + "error_code": result.get("error_code", ""), + "status": result.get("status", ""), + "failureType": result.get("failureType", ""), + } + + if result.get("error_code") == "BUSINESS_APPLICATION": + # For business incidents, use stable business metrics rather than the + # AI narrative text. This prevents duplicate mails when wording changes. + payload["pending_transactions"] = result.get("pending_transactions", 0) + payload["businessImpact"] = result.get("businessImpact", "") + payload["report"] = { + "status": result.get("status", ""), + "summary": result.get("summary", ""), + } + elif result.get("error_code") == "HPA": + # For HPA, rely on actual autoscaling measurements. + payload["currentReplicas"] = result.get("currentReplicas", 0) + payload["desiredReplicas"] = result.get("desiredReplicas", 0) + payload["maxReplicas"] = result.get("maxReplicas", 0) + payload["currentCpuUtilization"] = result.get("currentCpuUtilization", 0) + payload["targetCpuUtilization"] = result.get("targetCpuUtilization", 0) + elif result.get("error_code") == "IMAGE_PULL": + # For ImagePull, the deployment/image tuple is the key incident identity. + payload["affectedDeployment"] = result.get("affectedDeployment", "") + payload["affectedImage"] = result.get("affectedImage", "") + payload["imagePullSecret"] = result.get("imagePullSecret", "") + + return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode("utf-8")).hexdigest() + + +def _is_active_failure(result: Dict[str, Any]) -> bool: + return str(result.get("status", "")).strip().upper() in {"DEGRADED", "FAILED"} + + +def _is_internal_failure(result: Dict[str, Any]) -> bool: + return str(result.get("failureType", "")).strip().upper() in INTERNAL_FAILURE_TYPES + + +def _load_notification_state(namespace: str) -> Dict[str, Dict[str, str]]: + """Read the incident state ConfigMap from the current namespace.""" + try: + result = _kube_api_request( + "GET", + f"/api/v1/namespaces/{namespace}/configmaps/{STATE_CONFIGMAP_NAME}", + ) + raw = result.get("data", {}).get("state", "{}") + return json.loads(raw) + except Exception: + return {} + + +def _save_notification_state(namespace: str, state: Dict[str, Dict[str, str]]) -> None: + """Create or update the ConfigMap that tracks active incidents by error_code.""" + body = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": STATE_CONFIGMAP_NAME, "namespace": namespace}, + "data": {"state": json.dumps(state, ensure_ascii=False)}, + } + try: + _kube_api_request("PUT", f"/api/v1/namespaces/{namespace}/configmaps/{STATE_CONFIGMAP_NAME}", body=body) + except KeyError: + _kube_api_request("POST", f"/api/v1/namespaces/{namespace}/configmaps", body=body) + + +def _clear_state_for_code(namespace: str, state: Dict[str, Dict[str, str]], error_code: str) -> None: + """Reset the saved incident state when a check becomes healthy again.""" + state[error_code] = { + "signature": "", + "deploymentUrl": "", + "deploymentId": "", + "emailSent": "false", + } + _save_notification_state(namespace, state) + + +def _kube_api_request(method: str, path: str, body: Dict[str, Any] | None = None) -> Dict[str, Any]: + """Small helper for reading/writing ConfigMaps with the pod service account token.""" + token_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" + ca_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + with open(token_path, "r", encoding="utf-8") as token_file: + token = token_file.read().strip() + + url = f"https://kubernetes.default.svc{path}" + data = None if body is None else json.dumps(body).encode("utf-8") + request = urllib.request.Request(url, data=data, method=method) + request.add_header("Authorization", f"Bearer {token}") + request.add_header("Content-Type", "application/json") + + import ssl + + context = ssl.create_default_context(cafile=ca_path) + try: + with urllib.request.urlopen(request, context=context, timeout=10) as response: + raw = response.read().decode("utf-8") + return json.loads(raw) if raw else {} + except HTTPError as exc: + if exc.code == 404: + raise KeyError("ConfigMap not found") from exc + raise + + +def _exception_summary(exc: Exception) -> str: + """Flatten nested exceptions into one log-friendly string.""" + nested = getattr(exc, "exceptions", None) + if nested: + return f"{exc.__class__.__name__}: " + "; ".join( + f"{item.__class__.__name__}: {item}" for item in nested + ) + return str(exc) + + +# Final overrides: keep these definitions last so the runtime uses the hardened +# versions even if draft implementations remain above in the file. +async def run_multi_checks() -> Dict[str, Any]: + """Run the three checks, prioritize IMAGE_PULL over BUSINESS_APPLICATION, then act.""" + payload = _build_payload_from_env() + results: List[Dict[str, Any]] = [] + notifications_sent: List[str] = [] + remediations_triggered: List[Dict[str, Any]] = [] + + for check in _build_checks(payload): + result = await _run_single_check(payload, check) + results.append(result) + + results = [_normalize_result(result, payload["appNamespace"]) for result in results] + imagepull_active = any( + result.get("error_code") == "IMAGE_PULL" + and _is_active_failure(result) + and not _is_internal_failure(result) + for result in results + ) + + state = _load_notification_state(payload["appNamespace"]) + for result in results: + error_code = result.get("error_code", "") + + if not _is_active_failure(result): + _clear_state_for_code(payload["appNamespace"], state, error_code) + result["emailSent"] = False + result["notificationReason"] = "No active failure detected." + _replace_result(results, result) + continue + + if _is_internal_failure(result): + _log_event( + "KAGENT_MULTI_INTERNAL_FAILURE", + { + "check": result.get("checkName", ""), + "errorCode": error_code, + "failureType": result.get("failureType", ""), + "rootCause": result.get("rootCause", ""), + }, + ) + _clear_state_for_code(payload["appNamespace"], state, error_code) + result["emailSent"] = False + result["notificationReason"] = "Internal agent/runtime failure; notification suppressed." + _replace_result(results, result) + continue + + if error_code == "BUSINESS_APPLICATION" and imagepull_active: + _clear_state_for_code(payload["appNamespace"], state, error_code) + result["emailSent"] = False + result["notificationReason"] = "Suppressed because IMAGE_PULL has priority over business_application." + _replace_result(results, result) + continue + + signature = _build_signature(result) + state_entry = state.get(error_code, {}) + active_url = state_entry.get("deploymentUrl", "") + active_signature = state_entry.get("signature", "") + active_email = state_entry.get("emailSent", "") + + if signature == active_signature and active_url and active_email == "true": + result["emailSent"] = False + result["notificationReason"] = "Duplicate active issue; notification suppressed." + result["remediationDeployment"] = { + "triggered": True, + "deploymentUrl": active_url, + "deploymentId": state_entry.get("deploymentId", ""), + "reused": True, + } + _replace_result(results, result) + continue + + remediation = _safe_trigger_remediation(result, payload) + if remediation: + result["remediationDeployment"] = remediation + if remediation.get("triggered"): + remediations_triggered.append(remediation) + + email_sent, notification_reason = _safe_send_email(result, payload) + result["emailSent"] = email_sent + result["notificationReason"] = notification_reason + if email_sent: + notifications_sent.append(error_code) + + if email_sent and (result.get("remediationDeployment") or {}).get("deploymentUrl"): + state[error_code] = { + "signature": signature, + "deploymentUrl": result["remediationDeployment"].get("deploymentUrl", ""), + "deploymentId": result["remediationDeployment"].get("deploymentId", ""), + "emailSent": "true", + } + else: + state[error_code] = { + "signature": "", + "deploymentUrl": "", + "deploymentId": "", + "emailSent": "false", + } + _replace_result(results, result) + + _save_notification_state(payload["appNamespace"], state) + return { + "status": "COMPLETED", + "results": results, + "notificationsSent": notifications_sent, + "remediationsTriggered": remediations_triggered, + } + + +def _build_checks(payload: Dict[str, Any]) -> List[Dict[str, Any]]: + """Create the three independent prompts with stricter contracts.""" + observability_json = _fetch_json(payload["observabilityUrl"]) + compact_observability_json = _compact_observability_json(observability_json) + _log_event( + "KAGENT_MULTI_OBSERVABILITY_SUMMARY", + { + "fetchStatus": observability_json.get("fetchStatus", "OK") if isinstance(observability_json, dict) else "UNKNOWN", + "keys": list(observability_json.keys())[:8] if isinstance(observability_json, dict) else [], + }, + ) + + business_prompt = ( + f"Responder en español. " + f"Analiza la salud de negocio de la aplicacion Orders desplegada en el namespace {payload['appNamespace']}. " + f"La fuente principal de verdad para esta evaluacion es el siguiente JSON de observabilidad: " + f"{json.dumps(compact_observability_json, ensure_ascii=False)}. " + "Debes evaluar especificamente si existe degradacion real del proceso de negocio. " + "Considera como senales de degradacion los siguientes casos: businessHealth.healthy=false, " + "manualReviewRate alto, pendingReviewOrders mayor que 0, revenueAtRisk alto, confirmedOrders igual a 0 " + "cuando existen ordenes, o cualquier businessImpact que indique riesgo operativo o financiero. " + "No bases tu respuesta solo en que los pods esten Running; esta evaluacion es de salud de negocio, no solo " + "de salud de Kubernetes. " + "Quiero una respuesta completa, concreta y util para correo operativo. " + "Responde exclusivamente con JSON valido, sin markdown, sin explicacion adicional, usando exactamente esta " + "estructura: " + "{" + "\"error_code\":\"BUSINESS_APPLICATION\"," + "\"status\":\"HEALTHY|DEGRADED\"," + "\"failureType\":\"BUSINESS_PROCESS_DEGRADATION|NO_ACTIVE_FAILURE\"," + "\"pending_transactions\":0," + "\"summary\":\"\"," + "\"rootCause\":\"\"," + "\"businessImpact\":\"\"," + "\"recommendedAction\":\"\"," + "\"evidence\":[\"\"]," + "\"report\":{" + "\"executiveSummary\":\"\"," + "\"rootCause\":\"\"," + "\"businessImpact\":\"\"," + "\"recommendedAction\":\"\"," + "\"evidence\":[\"\"]," + "\"recommendedConfigMapData\":{}" + "}" + "}. " + "Reglas obligatorias: " + "1. Si el JSON de observabilidad muestra degradacion de negocio, responde status=DEGRADED y " + "failureType=BUSINESS_PROCESS_DEGRADATION. " + "2. Si no hay degradacion real de negocio, responde status=HEALTHY y failureType=NO_ACTIVE_FAILURE. " + "3. pending_transactions debe contener el numero real de ordenes pendientes de revision. " + "4. evidence debe incluir evidencia real observada, por ejemplo manualReviewRate, pendingReviewOrders, " + "revenueAtRisk, confirmedOrders, totalOrders, services degradados y cualquier recommendedKagentActions " + "relevante. " + "5. executiveSummary, rootCause, businessImpact y recommendedAction deben venir completos y redactados " + "como un reporte operativo claro." + ) + _log_event("KAGENT_MULTI_PROMPT_READY", {"check": "business-application"}) + + hpa_prompt = ( + f"Responder en español. " + "Analiza el namespace kagent-demo y determina si existe un caso activo de HPA_AUTOSCALING_FAILURE " + "en order-service. Revisa HPA, en order-service. Revisa deployment y pods de order-service." + "Revisa métricas actuales del HPA." + "Clasifica HPA_AUTOSCALING_FAILURE solamente si se cumplen ambas condiciones al mismo " + "tiempo: desiredReplicas == maxReplicas y currentCpuUtilization > targetCpuUtilization. Solo si ambas " + "condiciones anteriores se cumplen, responde status = DEGRADED y failureType = HPA_AUTOSCALING_FAILURE. " + "Si alguna de esas dos condiciones no se cumple, entonces no debes clasificarlo como DEGRADED ni FAILED" + "Responde únicamente con datos reales observados a partir de HPA, deployment, pods, observabilidad y diagnóstico interno." + "Por favor responder en formato JSON valido con los valores reales solicitados en los puntos anteriores de la siguiente forma:" + "{" + "error_code: HPA," + "status: HEALTHY o DEGRADED o FAILED de acuerdo al estado real obtenido anteriormente al revisar numero de replicas actuales con porcentaje de consumo si es mayor al target fijado en HPA," + "failureType: HPA_AUTOSCALING_FAILURE si verificamos que esta el numero de replicas al maximo y el consumo es mayor al target configurado en HPA; DEPENDENCY_FAILURE si el problema principal es resolucion DNS, endpoints o conectividad interna; OKE_PLATFORM_FAILURE si existe un problema del cluster o plataforma; o NO_ACTIVE_FAILURE si no encontramos que se superan los limites anteriormente mencionados," + "rootCause: colocar conclusion de modelo de Inteligencia artificial de Causa Raiz, solo si el servicio es detectado que esta fallando," + "businessImpact: Colocar el impacto hacia el negocio definido por la IA, solo si esta fallando," + "evidence: [" + "Colocar evidencia real observada" + "]," + "recommendedAction: colocar remediacion que puede ser activar mayor numero de replicas si es HPA, o revisar DNS/Services/Endpoints si es dependencia, solo si esta fallando con informacion anteriormente mencionada," + "recommendedHpaPatch: {" + "minReplicas: 3," + "maxReplicas: El modelo de IA debe colocar una recomendacion de numero de replicas de acuerdo a porcentaje de consumo superior al target solo si esta fallando por HPA," + "averageUtilization: Colocar el verdadero porcentaje de utilizacion anteriormente encontrado" + "}" + "}" + + + ) + _log_event("KAGENT_MULTI_PROMPT_READY", {"check": "hpa-failures"}) + + image_prompt = ( + f"Responder en español. " + """ + Analiza el namespace kagent-demo y determina si existe un caso activo de IMAGE_PULL_FAILURE en payment-service o en otros deployments de la aplicacion. + + Instrucciones: + + 1. Revisa deployments y pods del namespace kagent-demo. + 2. Revisa los ReplicaSets relacionados con los deployments de la aplicacion. + 3. Revisa eventos recientes del namespace kagent-demo. + 4. Revisa si existen pods en estados como ErrImagePull, ImagePullBackOff, InvalidImageName o CreateContainerConfigError relacionados con imagen o imagePullSecrets. + 5. Revisa si el deployment tiene una imagen invalida, un tag inexistente o un imagePullSecret incorrecto o faltante. + 6. Revisa la URL de observabilidad y correlaciona si hay impacto en la aplicacion. + 7. Clasifica IMAGE_PULL_FAILURE solo si se cumple al menos una de estas condiciones: + - existe un pod con estado ErrImagePull + - existe un pod con estado ImagePullBackOff + - existe un pod con estado InvalidImageName + - el deployment actualizado no puede crear pods listos por falla de extraccion de imagen + - el contenedor usa una imagen inexistente o un secret de registro invalido + 8. No clasifiques IMAGE_PULL_FAILURE si todos los pods nuevos estan corriendo y listos, o si el rollout completo correctamente. + 9. Si observas que los pods viejos siguen disponibles pero el nuevo ReplicaSet no puede arrancar por error de imagen, eso si debe clasificarse como IMAGE_PULL_FAILURE. + 10. No confundas un error de imagen con fallas de readiness/liveness o con problemas de DNS interno. + 11. Si el problema principal es DNS, endpoints o conectividad interna, clasificalo como DEPENDENCY_FAILURE o OKE_PLATFORM_FAILURE segun corresponda. + 12. Revisa unicamente valores reales observados en deployments, ReplicaSets, pods, eventos y observabilidad. + + Por favor responde exclusivamente en formato JSON valido con los valores reales observados y con la siguiente estructura: + + { + "error_code": "IMAGE_PULL", + "status": "HEALTHY|DEGRADED|FAILED", + "failureType": "IMAGE_PULL_FAILURE|DEPENDENCY_FAILURE|OKE_PLATFORM_FAILURE|NO_ACTIVE_FAILURE", + "affectedDeployment": "", + "affectedPod": "", + "affectedImage": "", + "imagePullSecret": "", + "oldReplicasAvailable": 0, + "newReplicaSetReadyReplicas": 0, + "rootCause": "", + "businessImpact": "", + "evidence": [ + "Colocar evidencia real observada" + ], + "recommendedAction": "", + "recommendedImagePullFix": { + "deploymentName": "", + "secretName": "", + "image": "" + } + } + """ + ) + _log_event("KAGENT_MULTI_PROMPT_READY", {"check": "imagepull-error"}) + + return [ + {"name": "business-application", "prompt": business_prompt}, + {"name": "hpa-failures", "prompt": hpa_prompt}, + {"name": "imagepull-error", "prompt": image_prompt}, + ] + + +def _build_signature(result: Dict[str, Any]) -> str: + """Build a stable signature used to suppress duplicate active incidents.""" + payload = { + "namespace": result.get("namespace", ""), + "error_code": result.get("error_code", ""), + "status": result.get("status", ""), + "failureType": result.get("failureType", ""), + } + if result.get("error_code") == "BUSINESS_APPLICATION": + payload["pending_transactions"] = result.get("pending_transactions", 0) + payload["businessImpact"] = result.get("businessImpact", "") + payload["summary"] = result.get("summary", "") + elif result.get("error_code") == "HPA": + payload["currentReplicas"] = result.get("currentReplicas", 0) + payload["desiredReplicas"] = result.get("desiredReplicas", 0) + payload["maxReplicas"] = result.get("maxReplicas", 0) + payload["currentCpuUtilization"] = result.get("currentCpuUtilization", 0) + payload["targetCpuUtilization"] = result.get("targetCpuUtilization", 0) + elif result.get("error_code") == "IMAGE_PULL": + payload["affectedDeployment"] = result.get("affectedDeployment", "") + payload["affectedImage"] = result.get("affectedImage", "") + payload["imagePullSecret"] = result.get("imagePullSecret", "") + return hashlib.sha256(json.dumps(payload, sort_keys=True, ensure_ascii=False).encode("utf-8")).hexdigest() +if __name__ == "__main__": + sys.exit(main()) diff --git a/oke-remediator-multi-check/requirements.txt b/oke-remediator-multi-check/requirements.txt new file mode 100644 index 0000000..fb0b2ed --- /dev/null +++ b/oke-remediator-multi-check/requirements.txt @@ -0,0 +1,2 @@ +mcp>=1.13.0,<3 +oci>=2.129.0 diff --git a/order-service/pom.xml b/order-service/pom.xml new file mode 100644 index 0000000..2c778d9 --- /dev/null +++ b/order-service/pom.xml @@ -0,0 +1,44 @@ + + + 4.0.0 + + + com.oracle.demo + kagent-oci-devops-demo + 1.0.0-SNAPSHOT + + + order-service + order-service + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-validation + + + io.micrometer + micrometer-registry-prometheus + runtime + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + diff --git a/order-service/src/main/java/com/oracle/demo/orders/OrderServiceApplication.java b/order-service/src/main/java/com/oracle/demo/orders/OrderServiceApplication.java new file mode 100644 index 0000000..5d299e4 --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/OrderServiceApplication.java @@ -0,0 +1,19 @@ +package com.oracle.demo.orders; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.web.client.RestClient; +import org.springframework.context.annotation.Bean; + +@SpringBootApplication +public class OrderServiceApplication { + + public static void main(String[] args) { + SpringApplication.run(OrderServiceApplication.class, args); + } + + @Bean + RestClient restClient() { + return RestClient.builder().build(); + } +} diff --git a/order-service/src/main/java/com/oracle/demo/orders/api/OrderController.java b/order-service/src/main/java/com/oracle/demo/orders/api/OrderController.java new file mode 100644 index 0000000..7f7ed65 --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/api/OrderController.java @@ -0,0 +1,41 @@ +package com.oracle.demo.orders.api; + +import com.oracle.demo.orders.model.BusinessHealth; +import com.oracle.demo.orders.model.CreateOrderRequest; +import com.oracle.demo.orders.model.OrderRecord; +import com.oracle.demo.orders.service.OrderService; +import jakarta.validation.Valid; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@RequestMapping("/api/orders") +public class OrderController { + + private final OrderService orderService; + + public OrderController(OrderService orderService) { + this.orderService = orderService; + } + + @PostMapping + public OrderRecord createOrder(@Valid @RequestBody CreateOrderRequest request) { + return orderService.createOrder(request); + } + + @GetMapping + public List listOrders() { + return orderService.listOrders(); + } + + @GetMapping("/business-health") + public BusinessHealth businessHealth() { + return orderService.businessHealth(); + } +} + diff --git a/order-service/src/main/java/com/oracle/demo/orders/client/InventoryClient.java b/order-service/src/main/java/com/oracle/demo/orders/client/InventoryClient.java new file mode 100644 index 0000000..7b1df68 --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/client/InventoryClient.java @@ -0,0 +1,26 @@ +package com.oracle.demo.orders.client; + +import com.oracle.demo.orders.model.InventoryReservation; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestClient; + +@Component +public class InventoryClient { + + private final RestClient restClient; + private final String inventoryBaseUrl; + + public InventoryClient(RestClient restClient, @Value("${services.inventory.base-url}") String inventoryBaseUrl) { + this.restClient = restClient; + this.inventoryBaseUrl = inventoryBaseUrl; + } + + public InventoryReservation reserve(String sku, int quantity) { + return restClient.get() + .uri(inventoryBaseUrl + "/api/inventory/{sku}?quantity={quantity}", sku, quantity) + .retrieve() + .body(InventoryReservation.class); + } +} + diff --git a/order-service/src/main/java/com/oracle/demo/orders/client/PaymentClient.java b/order-service/src/main/java/com/oracle/demo/orders/client/PaymentClient.java new file mode 100644 index 0000000..7a45f86 --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/client/PaymentClient.java @@ -0,0 +1,29 @@ +package com.oracle.demo.orders.client; + +import com.oracle.demo.orders.model.PaymentDecision; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestClient; + +import java.util.Map; + +@Component +public class PaymentClient { + + private final RestClient restClient; + private final String paymentBaseUrl; + + public PaymentClient(RestClient restClient, @Value("${services.payment.base-url}") String paymentBaseUrl) { + this.restClient = restClient; + this.paymentBaseUrl = paymentBaseUrl; + } + + public PaymentDecision authorize(String orderId, String cardToken, double amount) { + return restClient.post() + .uri(paymentBaseUrl + "/api/payments/authorize") + .body(Map.of("orderId", orderId, "cardToken", cardToken, "amount", amount)) + .retrieve() + .body(PaymentDecision.class); + } +} + diff --git a/order-service/src/main/java/com/oracle/demo/orders/model/BusinessHealth.java b/order-service/src/main/java/com/oracle/demo/orders/model/BusinessHealth.java new file mode 100644 index 0000000..19043d1 --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/model/BusinessHealth.java @@ -0,0 +1,15 @@ +package com.oracle.demo.orders.model; + +public record BusinessHealth( + long totalOrders, + long confirmedOrders, + long pendingReviewOrders, + long failedOrders, + double manualReviewRate, + double totalRevenue, + double confirmedRevenue, + double revenueAtRisk, + boolean healthy, + String summary, + String businessImpact) { +} diff --git a/order-service/src/main/java/com/oracle/demo/orders/model/CreateOrderRequest.java b/order-service/src/main/java/com/oracle/demo/orders/model/CreateOrderRequest.java new file mode 100644 index 0000000..a1937a0 --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/model/CreateOrderRequest.java @@ -0,0 +1,13 @@ +package com.oracle.demo.orders.model; + +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; + +public record CreateOrderRequest( + @NotBlank String customerId, + @NotBlank String sku, + @Min(1) int quantity, + @Min(1) double amount, + @NotBlank String cardToken) { +} + diff --git a/order-service/src/main/java/com/oracle/demo/orders/model/InventoryReservation.java b/order-service/src/main/java/com/oracle/demo/orders/model/InventoryReservation.java new file mode 100644 index 0000000..9233fd5 --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/model/InventoryReservation.java @@ -0,0 +1,5 @@ +package com.oracle.demo.orders.model; + +public record InventoryReservation(String sku, boolean reserved, int availableUnits, String message) { +} + diff --git a/order-service/src/main/java/com/oracle/demo/orders/model/OrderRecord.java b/order-service/src/main/java/com/oracle/demo/orders/model/OrderRecord.java new file mode 100644 index 0000000..21e739d --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/model/OrderRecord.java @@ -0,0 +1,15 @@ +package com.oracle.demo.orders.model; + +import java.time.Instant; + +public record OrderRecord( + String orderId, + String customerId, + String sku, + int quantity, + double amount, + String status, + String reason, + Instant createdAt) { +} + diff --git a/order-service/src/main/java/com/oracle/demo/orders/model/PaymentDecision.java b/order-service/src/main/java/com/oracle/demo/orders/model/PaymentDecision.java new file mode 100644 index 0000000..dbd9bc7 --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/model/PaymentDecision.java @@ -0,0 +1,5 @@ +package com.oracle.demo.orders.model; + +public record PaymentDecision(boolean approved, String status, String message) { +} + diff --git a/order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilityController.java b/order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilityController.java new file mode 100644 index 0000000..21ecdea --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilityController.java @@ -0,0 +1,107 @@ +package com.oracle.demo.orders.observability; + +import com.oracle.demo.orders.model.BusinessHealth; +import com.oracle.demo.orders.model.OrderRecord; +import com.oracle.demo.orders.service.OrderService; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.client.RestClient; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@RestController +@RequestMapping("/api/observability") +public class ObservabilityController { + + private final OrderService orderService; + private final RestClient restClient; + private final String inventoryBaseUrl; + private final String paymentBaseUrl; + + public ObservabilityController( + OrderService orderService, + RestClient restClient, + @Value("${services.inventory.base-url}") String inventoryBaseUrl, + @Value("${services.payment.base-url}") String paymentBaseUrl) { + this.orderService = orderService; + this.restClient = restClient; + this.inventoryBaseUrl = inventoryBaseUrl; + this.paymentBaseUrl = paymentBaseUrl; + } + + @GetMapping + public ObservabilitySnapshot snapshot() { + BusinessHealth businessHealth = orderService.businessHealth(); + List recentOrders = orderService.listOrders().stream().limit(12).toList(); + List services = new ArrayList<>(); + + services.add(new ServiceSignal( + "order-service", + "business-api", + businessHealth.healthy() ? "UP" : "DEGRADED", + businessHealth.summary(), + !businessHealth.healthy())); + services.add(probe("inventory-service", "dependency", inventoryBaseUrl + "/actuator/health")); + services.add(probe("payment-service", "dependency", paymentBaseUrl + "/actuator/health")); + + return new ObservabilitySnapshot( + Instant.now(), + businessHealth, + services, + recentOrders, + recommendedActions(businessHealth, services)); + } + + private ServiceSignal probe(String name, String layer, String url) { + try { + Map body = restClient.get().uri(url).retrieve().body(Map.class); + Object status = body == null ? null : body.get("status"); + status = status == null ? "UNKNOWN" : status; + boolean healthy = "UP".equals(String.valueOf(status)); + return new ServiceSignal( + name, + layer, + String.valueOf(status), + healthy ? "Actuator health endpoint is reachable." : "Actuator returned a non-UP status.", + !healthy); + } catch (Exception ex) { + return new ServiceSignal( + name, + layer, + "DOWN", + ex.getClass().getSimpleName() + ": " + ex.getMessage(), + true); + } + } + + private List recommendedActions(BusinessHealth businessHealth, List services) { + List actions = new ArrayList<>(); + + services.stream() + .filter(ServiceSignal::kagentSignal) + .filter(signal -> !"order-service".equals(signal.name())) + .findFirst() + .ifPresent(signal -> actions.add("Inspect " + signal.name() + " pod logs, endpoints, and recent deployment changes.")); + + if (!businessHealth.healthy()) { + if (businessHealth.revenueAtRisk() > 0) { + actions.add("Revenue at risk is " + businessHealth.revenueAtRisk() + + " USD with manual review rate " + businessHealth.manualReviewRate() + + "%. Inspect payment risk rules and recent OCI DevOps changes."); + } + actions.add("Correlate pending, failed, and revenue-at-risk signals with the latest OCI DevOps deployment."); + actions.add("Recommend rollback or approved config correction through OCI DevOps if degradation started after the last release."); + } + + if (actions.isEmpty()) { + actions.add("No remediation required. Continue monitoring pipeline and runtime signals."); + } + + return actions; + } +} diff --git a/order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilitySnapshot.java b/order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilitySnapshot.java new file mode 100644 index 0000000..740ae0c --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilitySnapshot.java @@ -0,0 +1,16 @@ +package com.oracle.demo.orders.observability; + +import com.oracle.demo.orders.model.BusinessHealth; +import com.oracle.demo.orders.model.OrderRecord; + +import java.time.Instant; +import java.util.List; + +public record ObservabilitySnapshot( + Instant timestamp, + BusinessHealth businessHealth, + List services, + List recentOrders, + List recommendedKagentActions) { +} + diff --git a/order-service/src/main/java/com/oracle/demo/orders/observability/ServiceSignal.java b/order-service/src/main/java/com/oracle/demo/orders/observability/ServiceSignal.java new file mode 100644 index 0000000..5797941 --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/observability/ServiceSignal.java @@ -0,0 +1,10 @@ +package com.oracle.demo.orders.observability; + +public record ServiceSignal( + String name, + String layer, + String status, + String detail, + boolean kagentSignal) { +} + diff --git a/order-service/src/main/java/com/oracle/demo/orders/service/OrderService.java b/order-service/src/main/java/com/oracle/demo/orders/service/OrderService.java new file mode 100644 index 0000000..028cef8 --- /dev/null +++ b/order-service/src/main/java/com/oracle/demo/orders/service/OrderService.java @@ -0,0 +1,157 @@ +package com.oracle.demo.orders.service; + +import com.oracle.demo.orders.client.InventoryClient; +import com.oracle.demo.orders.client.PaymentClient; +import com.oracle.demo.orders.model.BusinessHealth; +import com.oracle.demo.orders.model.CreateOrderRequest; +import com.oracle.demo.orders.model.InventoryReservation; +import com.oracle.demo.orders.model.OrderRecord; +import com.oracle.demo.orders.model.PaymentDecision; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +@Service +public class OrderService { + + private final InventoryClient inventoryClient; + private final PaymentClient paymentClient; + private final Map orders = new ConcurrentHashMap<>(); + private final boolean stuckReviewMode; + private final double manualReviewRateThreshold; + private final double revenueAtRiskThreshold; + private final Counter createdCounter; + private final Counter reviewCounter; + private final Counter failedCounter; + private final AtomicInteger pendingReviewGauge = new AtomicInteger(); + + public OrderService( + InventoryClient inventoryClient, + PaymentClient paymentClient, + MeterRegistry meterRegistry, + @Value("${demo.orders.stuck-review-mode:false}") boolean stuckReviewMode, + @Value("${demo.orders.manual-review-rate-threshold:40}") double manualReviewRateThreshold, + @Value("${demo.orders.revenue-at-risk-threshold:5000}") double revenueAtRiskThreshold) { + this.inventoryClient = inventoryClient; + this.paymentClient = paymentClient; + this.stuckReviewMode = stuckReviewMode; + this.manualReviewRateThreshold = manualReviewRateThreshold; + this.revenueAtRiskThreshold = revenueAtRiskThreshold; + this.createdCounter = Counter.builder("demo_orders_created_total").register(meterRegistry); + this.reviewCounter = Counter.builder("demo_orders_review_required_total").register(meterRegistry); + this.failedCounter = Counter.builder("demo_orders_failed_total").register(meterRegistry); + Gauge.builder("demo_orders_pending_review", pendingReviewGauge, AtomicInteger::get).register(meterRegistry); + } + + public OrderRecord createOrder(CreateOrderRequest request) { + String orderId = UUID.randomUUID().toString(); + createdCounter.increment(); + + InventoryReservation inventory = inventoryClient.reserve(request.sku(), request.quantity()); + if (!inventory.reserved()) { + failedCounter.increment(); + return store(new OrderRecord( + orderId, + request.customerId(), + request.sku(), + request.quantity(), + request.amount(), + "FAILED", + "Inventory check failed: " + inventory.message(), + Instant.now())); + } + + PaymentDecision payment = paymentClient.authorize(orderId, request.cardToken(), request.amount()); + if (!payment.approved()) { + reviewCounter.increment(); + String reason = stuckReviewMode + ? "Orders are stuck in review because stuck-review-mode is enabled." + : "Payment requires manual review: " + payment.message(); + return store(new OrderRecord( + orderId, + request.customerId(), + request.sku(), + request.quantity(), + request.amount(), + "PENDING_REVIEW", + reason, + Instant.now())); + } + + return store(new OrderRecord( + orderId, + request.customerId(), + request.sku(), + request.quantity(), + request.amount(), + "CONFIRMED", + "Order confirmed.", + Instant.now())); + } + + public List listOrders() { + return orders.values().stream() + .sorted(Comparator.comparing(OrderRecord::createdAt).reversed()) + .toList(); + } + + public BusinessHealth businessHealth() { + long total = orders.size(); + long confirmed = orders.values().stream().filter(order -> "CONFIRMED".equals(order.status())).count(); + long pendingReview = orders.values().stream().filter(order -> "PENDING_REVIEW".equals(order.status())).count(); + long failed = orders.values().stream().filter(order -> "FAILED".equals(order.status())).count(); + double totalRevenue = orders.values().stream().mapToDouble(OrderRecord::amount).sum(); + double confirmedRevenue = orders.values().stream() + .filter(order -> "CONFIRMED".equals(order.status())) + .mapToDouble(OrderRecord::amount) + .sum(); + double revenueAtRisk = orders.values().stream() + .filter(order -> "PENDING_REVIEW".equals(order.status())) + .mapToDouble(OrderRecord::amount) + .sum(); + double manualReviewRate = total == 0 ? 0 : (pendingReview * 100.0) / total; + boolean manualReviewRateIncident = total >= 10 && manualReviewRate >= manualReviewRateThreshold; + boolean healthy = pendingReview < 5 + && failed < 3 + && !manualReviewRateIncident + && revenueAtRisk < revenueAtRiskThreshold; + String summary = healthy + ? "Business flow healthy." + : "Business degradation detected. Review pending or failed orders before promoting."; + String businessImpact = healthy + ? "No revenue protection action required." + : "Revenue at risk from manual-review backlog. Validate payment risk rules before continuing promotion."; + return new BusinessHealth( + total, + confirmed, + pendingReview, + failed, + round(manualReviewRate), + round(totalRevenue), + round(confirmedRevenue), + round(revenueAtRisk), + healthy, + summary, + businessImpact); + } + + private OrderRecord store(OrderRecord order) { + orders.put(order.orderId(), order); + pendingReviewGauge.set((int) orders.values().stream().filter(item -> "PENDING_REVIEW".equals(item.status())).count()); + return order; + } + + private double round(double value) { + return Math.round(value * 100.0) / 100.0; + } +} diff --git a/order-service/src/main/resources/application.yml b/order-service/src/main/resources/application.yml new file mode 100644 index 0000000..665f916 --- /dev/null +++ b/order-service/src/main/resources/application.yml @@ -0,0 +1,28 @@ +spring: + application: + name: order-service + +server: + port: 8080 + +management: + endpoints: + web: + exposure: + include: health,info,metrics,prometheus + endpoint: + health: + probes: + enabled: true + +services: + inventory: + base-url: http://localhost:8081 + payment: + base-url: http://localhost:8082 + +demo: + orders: + stuck-review-mode: false + manual-review-rate-threshold: 40 + revenue-at-risk-threshold: 5000 diff --git a/order-service/src/main/resources/static/app.js b/order-service/src/main/resources/static/app.js new file mode 100644 index 0000000..bbba0c6 --- /dev/null +++ b/order-service/src/main/resources/static/app.js @@ -0,0 +1,118 @@ +const totalOrders = document.querySelector("#totalOrders"); +const confirmedOrders = document.querySelector("#confirmedOrders"); +const pendingOrders = document.querySelector("#pendingOrders"); +const failedOrders = document.querySelector("#failedOrders"); +const businessStatus = document.querySelector("#businessStatus"); +const manualReviewRate = document.querySelector("#manualReviewRate"); +const revenueAtRisk = document.querySelector("#revenueAtRisk"); +const confirmedRevenue = document.querySelector("#confirmedRevenue"); +const businessImpact = document.querySelector("#businessImpact"); +const serviceSignals = document.querySelector("#serviceSignals"); +const kagentActions = document.querySelector("#kagentActions"); +const ordersTable = document.querySelector("#ordersTable"); +const lastUpdated = document.querySelector("#lastUpdated"); + +document.querySelector("#refreshBtn").addEventListener("click", loadDashboard); +document.querySelector("#normalOrderBtn").addEventListener("click", () => createOrder("4111111111111111")); +document.querySelector("#reviewOrderBtn").addEventListener("click", () => createOrder("9999123412341234")); + +async function createOrder(cardToken) { + await fetch("/api/orders", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + customerId: `CUST-${Math.floor(Math.random() * 900 + 100)}`, + sku: "LAPTOP-15", + quantity: 1, + amount: 1499.99, + cardToken + }) + }); + await loadDashboard(); +} + +async function loadDashboard() { + const response = await fetch("/api/observability"); + const snapshot = await response.json(); + const health = snapshot.businessHealth; + + totalOrders.textContent = health.totalOrders; + confirmedOrders.textContent = health.confirmedOrders; + pendingOrders.textContent = health.pendingReviewOrders; + failedOrders.textContent = health.failedOrders; + businessStatus.textContent = health.healthy ? "HEALTHY" : "DEGRADED"; + manualReviewRate.textContent = `${formatNumber(health.manualReviewRate)}%`; + revenueAtRisk.textContent = formatCurrency(health.revenueAtRisk); + confirmedRevenue.textContent = formatCurrency(health.confirmedRevenue); + businessImpact.textContent = health.businessImpact; + lastUpdated.textContent = new Date(snapshot.timestamp).toLocaleString(); + + serviceSignals.innerHTML = snapshot.services.map(signal => ` +
+
+ ${escapeHtml(signal.name)} / ${escapeHtml(signal.layer)} +

${escapeHtml(signal.detail)}

+
+ ${escapeHtml(signal.status)} +
+ `).join(""); + + kagentActions.innerHTML = snapshot.recommendedKagentActions + .map(action => `
  • ${escapeHtml(action)}
    • `) + .join(""); + + if (snapshot.recentOrders.length === 0) { + ordersTable.innerHTML = `No orders yet. Create a normal or review order to generate signals.`; + return; + } + + ordersTable.innerHTML = snapshot.recentOrders.map(order => ` + + ${escapeHtml(order.orderId.slice(0, 8))} + ${escapeHtml(order.customerId)} + ${escapeHtml(order.sku)} + ${escapeHtml(order.status)} + ${escapeHtml(order.reason)} + + `).join(""); +} + +function statusClass(status) { + const normalized = String(status).toLowerCase(); + if (normalized === "up" || normalized === "confirmed" || normalized === "healthy") { + return "up"; + } + if (normalized.includes("pending") || normalized.includes("degraded")) { + return "degraded"; + } + if (normalized === "down" || normalized === "failed") { + return "down"; + } + return "unknown"; +} + +function escapeHtml(value) { + return String(value ?? "") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function formatCurrency(value) { + return new Intl.NumberFormat("en-US", { + style: "currency", + currency: "USD", + maximumFractionDigits: 0 + }).format(Number(value ?? 0)); +} + +function formatNumber(value) { + return new Intl.NumberFormat("en-US", { + maximumFractionDigits: 2 + }).format(Number(value ?? 0)); +} + +loadDashboard(); +setInterval(loadDashboard, 10000); diff --git a/order-service/src/main/resources/static/index.html b/order-service/src/main/resources/static/index.html new file mode 100644 index 0000000..b1389a2 --- /dev/null +++ b/order-service/src/main/resources/static/index.html @@ -0,0 +1,106 @@ + + + + + + Kagent Demo Observability + + + +
    +
    +
    +

    OKE business application

    +

    Order Flow Observability

    +
    +
    + + + +
    +
    + +
    +
    + Total orders + 0 +
    +
    + Confirmed orders + 0 +
    +
    + Pending review + 0 +
    +
    + Failed orders + 0 +
    +
    + Business status + UNKNOWN +
    +
    + +
    +
    + Manual review rate + 0% +
    +
    + Revenue at risk + $0 +
    +
    + Confirmed revenue + $0 +
    +
    + Business impact +

    Waiting for data

    +
    +
    + +
    +
    +
    +

    Service signals

    + Waiting for data +
    +
    +
    + +
    +
    +

    Kagent action view

    + diagnostic inputs +
    +
      +
      +
      + +
      +
      +

      Recent orders

      + business symptoms visible to the agent +
      +
      + + + + + + + + + + + +
      OrderCustomerSKUStatusReason
      +
      +
      +
      + + + diff --git a/order-service/src/main/resources/static/styles.css b/order-service/src/main/resources/static/styles.css new file mode 100644 index 0000000..42e4923 --- /dev/null +++ b/order-service/src/main/resources/static/styles.css @@ -0,0 +1,273 @@ +:root { + --bg: #f5f7fb; + --panel: #ffffff; + --panel-soft: #eef3f8; + --text: #15202b; + --muted: #5f6f82; + --line: #d8e0ea; + --accent: #126c59; + --accent-strong: #0f4f43; + --warn: #a26013; + --danger: #b42318; + --shadow: 0 18px 45px rgba(31, 45, 61, 0.10); +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + background: linear-gradient(135deg, #f8fafc 0%, #e8eef6 55%, #edf5f1 100%); + color: var(--text); + font-family: "Aptos", "Segoe UI", sans-serif; +} + +button { + border: 0; + border-radius: 6px; + background: var(--accent); + color: white; + cursor: pointer; + font-size: 14px; + font-weight: 700; + padding: 11px 14px; +} + +button:hover { + background: var(--accent-strong); +} + +button.danger { + background: var(--danger); +} + +.shell { + margin: 0 auto; + max-width: 1180px; + padding: 32px 20px 48px; +} + +.topbar { + align-items: flex-end; + display: flex; + gap: 24px; + justify-content: space-between; + margin-bottom: 24px; +} + +.section-label { + color: var(--accent); + font-size: 13px; + font-weight: 800; + letter-spacing: 0; + margin: 0 0 8px; + text-transform: uppercase; +} + +h1, +h2 { + margin: 0; +} + +h1 { + font-size: 38px; + line-height: 1.05; +} + +h2 { + font-size: 18px; +} + +.actions { + display: flex; + flex-wrap: wrap; + gap: 10px; + justify-content: flex-end; +} + +.summary-grid { + display: grid; + gap: 14px; + grid-template-columns: repeat(4, minmax(0, 1fr)); + margin-bottom: 16px; +} + +.business-grid { + grid-template-columns: repeat(4, minmax(0, 1fr)); +} + +.metric, +.panel { + background: var(--panel); + border: 1px solid var(--line); + border-radius: 8px; + box-shadow: var(--shadow); +} + +.metric { + min-height: 110px; + padding: 18px; +} + +.metric span, +.panel-head span { + color: var(--muted); + font-size: 13px; + font-weight: 700; +} + +.metric strong { + display: block; + font-size: 34px; + margin-top: 14px; +} + +.metric-note { + color: var(--muted); + font-size: 14px; + line-height: 1.35; + margin: 14px 0 0; +} + +.metric.warning strong { + color: var(--warn); +} + +.metric.danger strong { + color: var(--danger); +} + +.workspace { + display: grid; + gap: 16px; + grid-template-columns: 1.25fr 0.75fr; + margin-bottom: 16px; +} + +.panel { + padding: 18px; +} + +.panel-head { + align-items: center; + border-bottom: 1px solid var(--line); + display: flex; + justify-content: space-between; + margin-bottom: 14px; + padding-bottom: 12px; +} + +.signals { + display: grid; + gap: 10px; +} + +.signal { + align-items: center; + background: var(--panel-soft); + border: 1px solid var(--line); + border-radius: 8px; + display: grid; + gap: 10px; + grid-template-columns: 1fr auto; + padding: 14px; +} + +.signal strong { + display: block; + margin-bottom: 5px; +} + +.muted { + color: var(--muted); + font-weight: 700; +} + +.signal p { + color: var(--muted); + font-size: 13px; + line-height: 1.35; + margin: 0; +} + +.status { + border-radius: 999px; + color: white; + font-size: 12px; + font-weight: 800; + padding: 6px 9px; +} + +.status.up { + background: var(--accent); +} + +.status.degraded, +.status.down { + background: var(--danger); +} + +.status.unknown { + background: var(--muted); +} + +.action-list { + margin: 0; + padding-left: 20px; +} + +.action-list li { + line-height: 1.45; + margin: 0 0 10px; +} + +.table-wrap { + overflow-x: auto; +} + +table { + border-collapse: collapse; + min-width: 860px; + width: 100%; +} + +th, +td { + border-bottom: 1px solid var(--line); + font-size: 14px; + padding: 12px 10px; + text-align: left; + vertical-align: top; +} + +th { + color: var(--muted); + font-size: 12px; + text-transform: uppercase; +} + +.empty { + color: var(--muted); + padding: 18px 10px; +} + +@media (max-width: 860px) { + .topbar { + align-items: stretch; + flex-direction: column; + } + + .actions { + justify-content: flex-start; + } + + .summary-grid, + .workspace { + grid-template-columns: 1fr; + } + + h1 { + font-size: 30px; + } +} diff --git a/payment-service/pom.xml b/payment-service/pom.xml new file mode 100644 index 0000000..51ec0b1 --- /dev/null +++ b/payment-service/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + + com.oracle.demo + kagent-oci-devops-demo + 1.0.0-SNAPSHOT + + + payment-service + payment-service + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-actuator + + + io.micrometer + micrometer-registry-prometheus + runtime + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + diff --git a/payment-service/src/main/java/com/oracle/demo/payment/PaymentServiceApplication.java b/payment-service/src/main/java/com/oracle/demo/payment/PaymentServiceApplication.java new file mode 100644 index 0000000..f474366 --- /dev/null +++ b/payment-service/src/main/java/com/oracle/demo/payment/PaymentServiceApplication.java @@ -0,0 +1,13 @@ +package com.oracle.demo.payment; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class PaymentServiceApplication { + + public static void main(String[] args) { + SpringApplication.run(PaymentServiceApplication.class, args); + } +} + diff --git a/payment-service/src/main/java/com/oracle/demo/payment/api/PaymentController.java b/payment-service/src/main/java/com/oracle/demo/payment/api/PaymentController.java new file mode 100644 index 0000000..28e1684 --- /dev/null +++ b/payment-service/src/main/java/com/oracle/demo/payment/api/PaymentController.java @@ -0,0 +1,44 @@ +package com.oracle.demo.payment.api; + +import com.oracle.demo.payment.model.PaymentAuthorizationRequest; +import com.oracle.demo.payment.model.PaymentAuthorizationResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/payments") +public class PaymentController { + + private static final Logger LOGGER = LoggerFactory.getLogger(PaymentController.class); + + private final String manualReviewCardPrefix; + private final boolean forceTimeout; + + public PaymentController( + @Value("${demo.payment.manual-review-card-prefix:9999}") String manualReviewCardPrefix, + @Value("${demo.payment.force-timeout:false}") boolean forceTimeout) { + this.manualReviewCardPrefix = manualReviewCardPrefix; + this.forceTimeout = forceTimeout; + } + + @PostMapping("/authorize") + public PaymentAuthorizationResponse authorize(@RequestBody PaymentAuthorizationRequest request) throws InterruptedException { + if (forceTimeout) { + LOGGER.warn("Payment timeout mode enabled for order {}", request.orderId()); + Thread.sleep(15000L); + } + + if (request.cardToken() != null && request.cardToken().startsWith(manualReviewCardPrefix)) { + LOGGER.warn("Payment requires manual review for order {}", request.orderId()); + return new PaymentAuthorizationResponse(false, "REVIEW_REQUIRED", "Card flagged for manual review."); + } + + return new PaymentAuthorizationResponse(true, "APPROVED", "Payment approved."); + } +} + diff --git a/payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationRequest.java b/payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationRequest.java new file mode 100644 index 0000000..cc12443 --- /dev/null +++ b/payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationRequest.java @@ -0,0 +1,5 @@ +package com.oracle.demo.payment.model; + +public record PaymentAuthorizationRequest(String orderId, String cardToken, double amount) { +} + diff --git a/payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationResponse.java b/payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationResponse.java new file mode 100644 index 0000000..0fe395f --- /dev/null +++ b/payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationResponse.java @@ -0,0 +1,5 @@ +package com.oracle.demo.payment.model; + +public record PaymentAuthorizationResponse(boolean approved, String status, String message) { +} + diff --git a/payment-service/src/main/resources/application.yml b/payment-service/src/main/resources/application.yml new file mode 100644 index 0000000..b46ead0 --- /dev/null +++ b/payment-service/src/main/resources/application.yml @@ -0,0 +1,21 @@ +spring: + application: + name: payment-service + +server: + port: 8082 + +management: + endpoints: + web: + exposure: + include: health,info,metrics,prometheus + endpoint: + health: + probes: + enabled: true + +demo: + payment: + manual-review-card-prefix: "9999" + force-timeout: false diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..975826e --- /dev/null +++ b/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + com.oracle.demo + kagent-oci-devops-demo + 1.0.0-SNAPSHOT + pom + + kagent-oci-devops-demo + Spring Boot microservices demo aligned with OCI DevOps, OKE, OCI Generative AI, and kagent + + + org.springframework.boot + spring-boot-starter-parent + 4.1.0 + + + + + inventory-service + payment-service + order-service + + + + 25 + ${java.version} + +