adding files 2

2026-09-04 13:54:16 +00:00
parent b7ec9db2bc
commit 8b3cc8dd10
83 changed files with 5045 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,6 @@
+target/
+.idea/
+.vscode/
+*.iml
+*.log
+
--- a/OCI_DEVOPS_KAGENT_FUNCTION_FLOW.md
+++ b/OCI_DEVOPS_KAGENT_FUNCTION_FLOW.md
@@ -0,0 +1,205 @@
+# OCI DevOps + kagent + OCI Function Remediation Flow
+
+This runbook connects an OCI DevOps deployment pipeline with kagent through an OCI Function.
+
+## 1. Expose kagent MCP privately
+
+Edit the reserved private IP in:
+
+```text
+k8s/kagent/kagent-controller-private-lb.yaml
+```
+
+Replace:
+
+```text
+<reserved-private-ip>
+```
+
+with the OCI reserved private IP that belongs to the LoadBalancer subnet.
+
+Apply the private LoadBalancer service:
+
+```bash
+kubectl apply -f k8s/kagent/kagent-controller-private-lb.yaml
+kubectl get svc kagent-controller-private-lb -n kagent
+```
+
+Save the private IP and build the MCP URL:
+
+```text
+http://<private-kagent-lb-ip>:8083/mcp
+```
+
+The OCI Function subnet must be able to reach this private IP on TCP `8083`.
+
+If the Service was previously created without the reserved private IP annotation, delete and recreate it:
+
+```bash
+kubectl delete svc kagent-controller-private-lb -n kagent
+kubectl apply -f k8s/kagent/kagent-controller-private-lb.yaml
+```
+
+## 2. Deploy the OCI Function
+
+Function path:
+
+```text
+oci-functions/kagent-devops-remediator
+```
+
+Configure these Function variables:
+
+```text
+KAGENT_MCP_URL=http://<private-kagent-lb-ip>:8083/mcp
+KAGENT_AGENT_NAME=k8s-agent
+KAGENT_AGENT_NAMESPACE=kagent
+APP_NAMESPACE=kagent-demo
+ORDER_OBSERVABILITY_URL=http://<order-lb-ip>/api/observability
+```
+
+The Function calls kagent MCP tool `invoke_agent` and requests a strict JSON diagnosis.
+
+## 3. Add Function invocation to OCI DevOps
+
+In the deploy pipeline, add a stage after the OKE deployment:
+
+```text
+Deploy application to OKE
+  -> Wait 60 seconds
+  -> Invoke Function: kagent-devops-remediator
+  -> Approval: review kagent diagnosis
+  -> Shell: apply ConfigMap fix
+  -> Shell: validate business health
+```
+
+The Function returns:
+
+```json
+{
+  "status": "HEALTHY|DEGRADED|FAILED",
+  "failureType": "BUSINESS_PROCESS_DEGRADATION",
+  "recommendedAction": "APPLY_CONFIGMAP_FIX",
+  "recommendedConfigMapData": {
+    "ORDER_MANUAL_REVIEW_RATE_THRESHOLD": "70",
+    "ORDER_REVENUE_AT_RISK_THRESHOLD": "15000"
+  },
+  "deploymentsToRestart": ["order-service"]
+}
+```
+
+## 4. Apply the ConfigMap remediation
+
+Use this command spec as a Shell deploy artifact:
+
+```text
+k8s/fix-business-config-command-spec.yaml
+```
+
+It patches `demo-config` and restarts `order-service`.
+
+## 5. Validate the application
+
+Use this command spec after remediation:
+
+```text
+k8s/validate-business-health-command-spec.yaml
+```
+
+Before using it, replace:
+
+```text
+http://<order-lb-ip>/api/observability
+```
+
+with the current LoadBalancer URL.
+
+## Recommended pipeline behavior
+
+Use this decision model:
+
+```text
+CONTINUE              -> no remediation
+APPROVAL_REQUIRED    -> pause for human review
+APPLY_CONFIGMAP_FIX  -> approval, then run fix-business-config-command-spec.yaml
+ROLLBACK             -> run rollback stage or redeploy previous image tag
+REDEPLOY             -> rerun deploy stage with corrected artifact
+```
+
+For the conference demo, keep approval before remediation. It makes the control boundary clear: kagent diagnoses, OCI DevOps executes.
+
+## Multi-error demo scenario
+
+This demo can simulate three simultaneous issues:
+
+```text
+1. Business degradation:
+   ConfigMap thresholds make order-service report degraded business health.
+
+2. Image pull failure:
+   payment-service is patched with an invalid imagePullSecret.
+
+3. HPA ceiling:
+   order-service HPA is capped at 3 replicas.
+```
+
+Artifacts:
+
+```text
+k8s/kagent-multi-error-payload.json
+k8s/broken/simulate-three-errors-command-spec.yaml
+k8s/remediate-three-errors-command-spec.yaml
+k8s/fix-business-config-command-spec.yaml
+k8s/fix-imagepull-secret-command-spec.yaml
+k8s/fix-order-hpa-command-spec.yaml
+```
+
+Recommended OCI DevOps flow:
+
+```text
+Deploy app
+  -> Wait 60-90 seconds
+  -> Optional Shell: simulate-three-errors-command-spec.yaml
+  -> Invoke Function with k8s/kagent-multi-error-payload.json
+  -> Approval: review kagent output
+  -> Shell: remediate-three-errors-command-spec.yaml
+  -> Shell: validate-business-health-command-spec.yaml
+  -> Invoke Function again to validate no active failures
+```
+
+The generic Function payload asks kagent to return a list of problems:
+
+```json
+{
+  "status": "HEALTHY|DEGRADED|FAILED",
+  "summary": "",
+  "problems": [
+    {
+      "failureType": "BUSINESS_PROCESS_DEGRADATION|IMAGE_PULL_FAILURE|HPA_AUTOSCALING_FAILURE|NO_ACTIVE_FAILURE",
+      "rootCause": "",
+      "evidence": [],
+      "impact": "",
+      "recommendedAction": "CONTINUE|APPROVAL_REQUIRED|APPLY_CONFIGMAP_FIX|FIX_IMAGE_PULL_SECRET|PATCH_HPA|ROLLBACK|REDEPLOY",
+      "affectedResources": [],
+      "recommendedConfigMapData": {},
+      "recommendedHpaPatch": {},
+      "deploymentsToRestart": []
+    }
+  ],
+  "overallRecommendedAction": "CONTINUE|APPROVAL_REQUIRED|APPLY_CONFIGMAP_FIX|FIX_IMAGE_PULL_SECRET|PATCH_HPA|ROLLBACK|REDEPLOY"
+}
+```
+
+For a first demo, use one remediation stage:
+
+```text
+remediate-three-errors-command-spec.yaml
+```
+
+For a more advanced demo, split remediation by action:
+
+```text
+APPLY_CONFIGMAP_FIX     -> fix-business-config-command-spec.yaml
+FIX_IMAGE_PULL_SECRET   -> fix-imagepull-secret-command-spec.yaml
+PATCH_HPA               -> fix-order-hpa-command-spec.yaml
+```
--- a/OCI_DEVOPS_REMEDIATION_PIPELINES.md
+++ b/OCI_DEVOPS_REMEDIATION_PIPELINES.md
@@ -0,0 +1,225 @@
+# OCI DevOps Remediation Pipelines
+
+This demo uses one detection path and three independent remediation pipelines. The preferred remediation style is declarative Kubernetes manifests applied by OCI DevOps.
+
+## Detection flow
+
+Use the main deploy or diagnosis pipeline to invoke the kagent Function with:
+
+```text
+k8s/kagent-multi-error-payload.json
+```
+
+kagent can return one or more problems:
+
+```text
+BUSINESS_PROCESS_DEGRADATION
+IMAGE_PULL_FAILURE
+HPA_AUTOSCALING_FAILURE
+```
+
+OCI DevOps does not provide a simple native OR/branch based on the Function JSON. Use one of these patterns:
+
+```text
+Manual demo:
+Invoke Function -> Approval -> operator starts the correct remediation pipeline.
+
+Automated demo:
+Function parses kagent response and calls OCI DevOps API to start the correct remediation pipeline.
+```
+
+## Pipeline 1: Business application remediation
+
+Pipeline name:
+
+```text
+kagent-remediate-business-config
+```
+
+Purpose:
+
+```text
+Fix order-service business degradation caused by ConfigMap thresholds or stuck review mode.
+```
+
+Artifacts:
+
+```text
+Name: remediate-business-configmap
+Type: Kubernetes manifest
+Path: k8s/remediation/business-configmap.yaml
+
+Name: remediate-order-rollout-restart
+Type: Kubernetes manifest
+Path: k8s/remediation/order-rollout-restart.yaml
+```
+
+Recommended parameters:
+
+```text
+APP_NAMESPACE=kagent-demo
+CONFIGMAP_NAME=demo-config
+ORDER_MANUAL_REVIEW_RATE_THRESHOLD=70
+ORDER_REVENUE_AT_RISK_THRESHOLD=15000
+ORDER_STUCK_REVIEW_MODE=false
+PAYMENT_FORCE_TIMEOUT=false
+INVENTORY_FORCE_OUTAGE=false
+WAIT_TIMEOUT=300s
+```
+
+Stages:
+
+```text
+approval-remediate-business
+  -> apply-business-configmap
+  -> apply-order-rollout-restart
+  -> wait-after-business-fix
+  -> invoke-kagent-function-validate
+```
+
+Important:
+
+```text
+order-service reads ConfigMap values as environment variables.
+After applying business-configmap.yaml, apply order-rollout-restart.yaml with a new <restart-token>.
+```
+
+## Pipeline 2: ImagePull remediation
+
+Pipeline name:
+
+```text
+kagent-remediate-imagepull-secret
+```
+
+Purpose:
+
+```text
+Restore the valid OCIR imagePullSecret in any business deployment: order-service, payment-service, or inventory-service.
+```
+
+Artifacts:
+
+```text
+Name: remediate-imagepull-order
+Type: Kubernetes manifest
+Path: k8s/remediation/imagepull-order.yaml
+
+Name: remediate-imagepull-payment
+Type: Kubernetes manifest
+Path: k8s/remediation/imagepull-payment.yaml
+
+Name: remediate-imagepull-inventory
+Type: Kubernetes manifest
+Path: k8s/remediation/imagepull-inventory.yaml
+
+Name: remediate-imagepull-all-services
+Type: Kubernetes manifest
+Path: k8s/remediation/imagepull-all-services.yaml
+```
+
+Manifest placeholders:
+
+```text
+<region-key>
+<tenancy-namespace>
+<image-tag>
+<restart-token>
+```
+
+Allowed values:
+
+```text
+DEPLOYMENT_NAME=order-service
+DEPLOYMENT_NAME=payment-service
+DEPLOYMENT_NAME=inventory-service
+```
+
+When remediating another service, set `CONTAINER_NAME` to the same value as `DEPLOYMENT_NAME`.
+
+Stages:
+
+```text
+approval-remediate-imagepull
+  -> apply-imagepull-service-manifest
+  -> wait-after-imagepull-fix
+  -> invoke-kagent-function-validate
+```
+
+For exactly three remediation pipelines, use `imagepull-all-services.yaml`. It restores the imagePullSecret in all three services and avoids conditional branching.
+
+Create one pipeline per target service only if you want narrower remediation:
+
+```text
+kagent-remediate-imagepull-order
+kagent-remediate-imagepull-payment
+kagent-remediate-imagepull-inventory
+```
+
+Or create one pipeline and manually select the matching manifest artifact based on kagent output.
+
+## Pipeline 3: HPA capacity remediation
+
+Pipeline name:
+
+```text
+kagent-remediate-order-hpa
+```
+
+Purpose:
+
+```text
+Increase HPA capacity when order-service reaches maxReplicas and demand still requires more pods.
+```
+
+Artifact:
+
+```text
+Name: remediate-order-hpa-capacity
+Type: Kubernetes manifest
+Path: k8s/remediation/order-hpa-capacity.yaml
+```
+
+Recommended parameters:
+
+```text
+APP_NAMESPACE=kagent-demo
+HPA_NAME=order-service
+MIN_REPLICAS=3
+MAX_REPLICAS=8
+CPU_TARGET=65
+```
+
+Stages:
+
+```text
+approval-remediate-hpa
+  -> apply-order-hpa-capacity
+  -> wait-after-hpa-fix
+  -> invoke-kagent-function-validate
+```
+
+## Demo execution order
+
+Use this sequence for the conference:
+
+```text
+1. Run kagent-deployment-pipeline to deploy the healthy application.
+2. Run kagent-simulate-failures-pipeline to inject controlled failures.
+3. Run invoke-kagent-function with kagent-multi-error-payload.json.
+4. Review kagent output.
+5. Run one or more remediation pipelines:
+   - kagent-remediate-business-config
+   - kagent-remediate-imagepull-order/payment/inventory
+   - kagent-remediate-order-hpa
+6. Run final kagent validation.
+```
+
+## Mapping from kagent response to remediation pipeline
+
+```text
+BUSINESS_PROCESS_DEGRADATION -> kagent-remediate-business-config
+IMAGE_PULL_FAILURE           -> kagent-remediate-imagepull-secret
+HPA_AUTOSCALING_FAILURE      -> kagent-remediate-order-hpa
+NO_ACTIVE_FAILURE            -> no remediation
+```
--- a/build_spec.yaml
+++ b/build_spec.yaml
@@ -0,0 +1,54 @@
+version: 0.1
+component: build
+timeoutInSeconds: 1800
+shell: bash
+
+steps:
+  - type: Command
+    name: Validate pipeline parameters
+    command: |
+      echo "REGISTRY=${REGISTRY}"
+      echo "NAMESPACE=${NAMESPACE}"
+      echo "IMAGE_TAG=${IMAGE_TAG}"
+
+      if [ -z "${REGISTRY}" ] || [ -z "${NAMESPACE}" ] || [ -z "${IMAGE_TAG}" ]; then
+        echo "REGISTRY, NAMESPACE, and IMAGE_TAG are required"
+        exit 1
+      fi
+  - type: Command
+    name: Login to container registry
+    command: |
+      docker login -u ${NAMESPACE}/${DOMAIN}/${USER} -p ${AUTHTOKEN} ${REGISTRY}
+
+  - type: Command
+    name: Build container images
+    command: |
+      docker build -f docker/inventory-service.Dockerfile -t ${REGISTRY}/${NAMESPACE}/kagent-demo/inventory-service:${IMAGE_TAG} .
+      docker build -f docker/payment-service.Dockerfile -t ${REGISTRY}/${NAMESPACE}/kagent-demo/payment-service:${IMAGE_TAG} .
+      docker build -f docker/order-service.Dockerfile -t ${REGISTRY}/${NAMESPACE}/kagent-demo/order-service:${IMAGE_TAG} .
+      docker build -f docker/oke-remediator-multi-check.Dockerfile -t ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-multi-check:${IMAGE_TAG} .
+
+  - type: Command
+    name: Push container images
+    command: |
+      docker push ${REGISTRY}/${NAMESPACE}/kagent-demo/inventory-service:${IMAGE_TAG}
+      docker push ${REGISTRY}/${NAMESPACE}/kagent-demo/payment-service:${IMAGE_TAG}
+      docker push ${REGISTRY}/${NAMESPACE}/kagent-demo/order-service:${IMAGE_TAG}
+      docker push ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-multi-check:${IMAGE_TAG}
+
+outputArtifacts:
+  - name: inventory-image
+    type: DOCKER_IMAGE
+    location: ${REGISTRY}/${NAMESPACE}/kagent-demo/inventory-service:${IMAGE_TAG}
+
+  - name: payment-image
+    type: DOCKER_IMAGE
+    location: ${REGISTRY}/${NAMESPACE}/kagent-demo/payment-service:${IMAGE_TAG}
+
+  - name: order-image
+    type: DOCKER_IMAGE
+    location: ${REGISTRY}/${NAMESPACE}/kagent-demo/order-service:${IMAGE_TAG}
+
+  - name: oke-remediator-multi-image
+    type: DOCKER_IMAGE
+    location: ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-multi-check:${IMAGE_TAG}
--- a/docker/inventory-service.Dockerfile
+++ b/docker/inventory-service.Dockerfile
@@ -0,0 +1,23 @@
+FROM container-registry.oracle.com/graalvm/jdk:25 AS build
+
+WORKDIR /workspace
+
+RUN microdnf install -y maven && microdnf clean all
+
+COPY pom.xml .
+COPY payment-service/pom.xml payment-service/pom.xml
+COPY order-service/pom.xml order-service/pom.xml
+COPY inventory-service/pom.xml inventory-service/pom.xml
+COPY inventory-service/src inventory-service/src
+
+RUN mvn -pl inventory-service -am clean package -DskipTests
+
+FROM container-registry.oracle.com/graalvm/jdk:25
+
+WORKDIR /app
+
+COPY --from=build /workspace/inventory-service/target/inventory-service-1.0.0-SNAPSHOT.jar app.jar
+
+EXPOSE 8081
+
+ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-jar", "app.jar"]
--- a/docker/oke-remediator-job.Dockerfile
+++ b/docker/oke-remediator-job.Dockerfile
@@ -0,0 +1,13 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+COPY oke-remediator-job/requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+
+COPY oke-remediator-job/main.py /app/main.py
+
+ENTRYPOINT ["python", "/app/main.py"]
--- a/docker/oke-remediator-multi-check.Dockerfile
+++ b/docker/oke-remediator-multi-check.Dockerfile
@@ -0,0 +1,13 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+COPY oke-remediator-multi-check/requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+
+COPY oke-remediator-multi-check/main.py /app/main.py
+
+ENTRYPOINT ["python", "/app/main.py"]
--- a/docker/order-service.Dockerfile
+++ b/docker/order-service.Dockerfile
@@ -0,0 +1,23 @@
+FROM container-registry.oracle.com/graalvm/jdk:25 AS build
+
+WORKDIR /workspace
+
+RUN microdnf install -y maven && microdnf clean all
+
+COPY pom.xml .
+COPY inventory-service/pom.xml inventory-service/pom.xml
+COPY payment-service/pom.xml payment-service/pom.xml
+COPY order-service/pom.xml order-service/pom.xml
+COPY order-service/src order-service/src
+
+RUN mvn -pl order-service -am clean package -DskipTests
+
+FROM container-registry.oracle.com/graalvm/jdk:25
+
+WORKDIR /app
+
+COPY --from=build /workspace/order-service/target/order-service-1.0.0-SNAPSHOT.jar app.jar
+
+EXPOSE 8080
+
+ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-jar", "app.jar"]
--- a/docker/payment-service.Dockerfile
+++ b/docker/payment-service.Dockerfile
@@ -0,0 +1,23 @@
+FROM container-registry.oracle.com/graalvm/jdk:25 AS build
+
+WORKDIR /workspace
+
+RUN microdnf install -y maven && microdnf clean all
+
+COPY pom.xml .
+COPY inventory-service/pom.xml inventory-service/pom.xml
+COPY order-service/pom.xml order-service/pom.xml
+COPY payment-service/pom.xml payment-service/pom.xml
+COPY payment-service/src payment-service/src
+
+RUN mvn -pl payment-service -am clean package -DskipTests
+
+FROM container-registry.oracle.com/graalvm/jdk:25
+
+WORKDIR /app
+
+COPY --from=build /workspace/payment-service/target/payment-service-1.0.0-SNAPSHOT.jar app.jar
+
+EXPOSE 8082
+
+ENTRYPOINT ["java", "-XX:+UseContainerSupport", "-jar", "app.jar"]
--- a/inventory-service/pom.xml
+++ b/inventory-service/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.oracle.demo</groupId>
+        <artifactId>kagent-oci-devops-demo</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>inventory-service</artifactId>
+    <name>inventory-service</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-actuator</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-prometheus</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>
--- a/inventory-service/src/main/java/com/oracle/demo/inventory/InventoryServiceApplication.java
+++ b/inventory-service/src/main/java/com/oracle/demo/inventory/InventoryServiceApplication.java
@@ -0,0 +1,13 @@
+package com.oracle.demo.inventory;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class InventoryServiceApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(InventoryServiceApplication.class, args);
+    }
+}
+
--- a/inventory-service/src/main/java/com/oracle/demo/inventory/api/InventoryController.java
+++ b/inventory-service/src/main/java/com/oracle/demo/inventory/api/InventoryController.java
@@ -0,0 +1,40 @@
+package com.oracle.demo.inventory.api;
+
+import com.oracle.demo.inventory.model.InventoryResponse;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/inventory")
+public class InventoryController {
+
+    private final Map<String, Integer> stock = new HashMap<>();
+    private final boolean forceOutage;
+
+    public InventoryController(@Value("${demo.inventory.force-outage:false}") boolean forceOutage) {
+        this.forceOutage = forceOutage;
+        stock.put("LAPTOP-15", 25);
+        stock.put("MOUSE-WL", 100);
+        stock.put("MONITOR-27", 12);
+    }
+
+    @GetMapping("/{sku}")
+    public InventoryResponse getInventory(@PathVariable String sku, @RequestParam(defaultValue = "1") int quantity) {
+        if (forceOutage) {
+            return new InventoryResponse(sku, false, 0, "Inventory service is intentionally degraded for the demo.");
+        }
+
+        int available = stock.getOrDefault(sku, 0);
+        boolean reserved = available >= quantity;
+        String message = reserved ? "Stock reserved." : "Insufficient stock.";
+        return new InventoryResponse(sku, reserved, available, message);
+    }
+}
+
--- a/inventory-service/src/main/java/com/oracle/demo/inventory/model/InventoryResponse.java
+++ b/inventory-service/src/main/java/com/oracle/demo/inventory/model/InventoryResponse.java
@@ -0,0 +1,5 @@
+package com.oracle.demo.inventory.model;
+
+public record InventoryResponse(String sku, boolean reserved, int availableUnits, String message) {
+}
+
--- a/inventory-service/src/main/resources/application.yml
+++ b/inventory-service/src/main/resources/application.yml
@@ -0,0 +1,21 @@
+spring:
+  application:
+    name: inventory-service
+
+server:
+  port: 8081
+
+management:
+  endpoints:
+    web:
+      exposure:
+        include: health,info,metrics,prometheus
+  endpoint:
+    health:
+      probes:
+        enabled: true
+
+demo:
+  inventory:
+    force-outage: false
+
--- a/k8s/base/all.yaml
+++ b/k8s/base/all.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: List
+items: []
+
--- a/k8s/base/configmap.yaml
+++ b/k8s/base/configmap.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: demo-config
+  namespace: kagent-demo
+data:
+  INVENTORY_FORCE_OUTAGE: "false"
+  PAYMENT_FORCE_TIMEOUT: "false"
+  PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999"
+  ORDER_STUCK_REVIEW_MODE: "false"
+  ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "40"
+  ORDER_REVENUE_AT_RISK_THRESHOLD: "5000"
+  INVENTORY_BASE_URL: "http://inventory-service:8081"
+  PAYMENT_BASE_URL: "http://payment-service:8082"
--- a/k8s/base/hpa.yaml
+++ b/k8s/base/hpa.yaml
@@ -0,0 +1,22 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: order-service
+  namespace: kagent-demo
+  labels:
+    app: order-service
+    demo.oracle.com/remediation: hpa-capacity
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: order-service
+  minReplicas: 1 
+  maxReplicas: 1 
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 30 
--- a/k8s/base/inventory.yaml
+++ b/k8s/base/inventory.yaml
@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: inventory-service
+  namespace: kagent-demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: inventory-service
+  template:
+    metadata:
+      labels:
+        app: inventory-service
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret
+      containers:
+        - name: inventory-service
+          image: xxx.ocir.io/xxxxxxx/kagent-demo/inventory-service:2.0.0
+          ports:
+            - containerPort: 8081
+          env:
+            - name: DEMO_INVENTORY_FORCE_OUTAGE
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: INVENTORY_FORCE_OUTAGE
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8081
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8081
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: inventory-service
+  namespace: kagent-demo
+spec:
+  selector:
+    app: inventory-service
+  ports:
+    - port: 8081
+      targetPort: 8081
--- a/k8s/base/namespace.yaml
+++ b/k8s/base/namespace.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kagent-demo
+
--- a/k8s/base/order-loadbalancer.yaml
+++ b/k8s/base/order-loadbalancer.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: order-service-lb
+  namespace: kagent-demo
+  annotations:
+    service.beta.kubernetes.io/oci-load-balancer-shape: flexible
+    service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10"
+    service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "10"
+    oci.oraclecloud.com/reserved-ips: "40.233.17.25"
+spec:
+  type: LoadBalancer
+  selector:
+    app: order-service
+  ports:
+    - name: http
+      port: 80
+      targetPort: 8080
--- a/k8s/base/order.yaml
+++ b/k8s/base/order.yaml
@@ -0,0 +1,75 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: order-service
+  namespace: kagent-demo
+spec:
+  replicas: ${REPICLAS_ORDER}
+  selector:
+    matchLabels:
+      app: order-service
+  template:
+    metadata:
+      labels:
+        app: order-service
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret
+      containers:
+        - name: order-service
+          image: ${REGISTRY}/${NAMESPACE}/kagent-demo/order-service:${IMAGE_TAG}
+          ports:
+            - containerPort: 8080
+          resources:
+            requests:
+              cpu: "100m"
+              memory: "256Mi"
+            limits:
+              cpu: "500m"
+              memory: "512Mi"
+          env:
+            - name: SERVICES_INVENTORY_BASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: INVENTORY_BASE_URL
+            - name: SERVICES_PAYMENT_BASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_BASE_URL
+            - name: DEMO_ORDERS_STUCK_REVIEW_MODE
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_STUCK_REVIEW_MODE
+            - name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
+            - name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_REVENUE_AT_RISK_THRESHOLD
+          startupProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8080
+            failureThreshold: 30
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8080
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 3
--- a/k8s/base/payment.yaml
+++ b/k8s/base/payment.yaml
@@ -0,0 +1,41 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: payment-service
+  namespace: kagent-demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: payment-service
+  template:
+    metadata:
+      labels:
+        app: payment-service
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret
+      containers:
+        - name: payment-service
+          image: mty.ocir.io/xxxxxxx/kagent-demo/payment-service:2.0.0
+          ports:
+            - containerPort: 8082
+          env:
+            - name: DEMO_PAYMENT_FORCE_TIMEOUT
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_FORCE_TIMEOUT
+            - name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8082
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8082
--- a/k8s/broken/business-failure-configmap.yaml
+++ b/k8s/broken/business-failure-configmap.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: demo-config
+  namespace: kagent-demo
+data:
+  INVENTORY_FORCE_OUTAGE: "false"
+  PAYMENT_FORCE_TIMEOUT: "false"
+  PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999"
+  ORDER_STUCK_REVIEW_MODE: "true"
+  ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "40"
+  ORDER_REVENUE_AT_RISK_THRESHOLD: "5000"
+  INVENTORY_BASE_URL: "http://inventory-service:8081"
+  PAYMENT_BASE_URL: "http://payment-service:8082"
--- a/k8s/broken/imagepull-secret-failure-command-spec.yaml
+++ b/k8s/broken/imagepull-secret-failure-command-spec.yaml
@@ -0,0 +1,48 @@
+version: 0.1
+component: command
+timeoutInSeconds: 600
+shell: bash
+failImmediatelyOnError: true
+
+env:
+  variables:
+    APP_NAMESPACE: kagent-demo
+    DEPLOYMENT_NAME: payment-service
+    CONTAINER_NAME: payment-service
+    BROKEN_IMAGE_PULL_SECRET: missing-ocir-secret
+
+steps:
+  - type: Command
+    name: Simulate missing imagePullSecret
+    command: |
+      echo "Forcing ImagePullBackOff by replacing imagePullSecrets with ${BROKEN_IMAGE_PULL_SECRET}"
+
+      kubectl patch deployment "${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" \
+        --type merge \
+        -p "{
+          \"spec\": {
+            \"template\": {
+              \"spec\": {
+                \"imagePullSecrets\": [
+                  {\"name\": \"${BROKEN_IMAGE_PULL_SECRET}\"}
+                ],
+                \"containers\": [
+                  {
+                    \"name\": \"${CONTAINER_NAME}\",
+                    \"imagePullPolicy\": \"Always\"
+                  }
+                ]
+              }
+            }
+          }
+        }"
+
+      kubectl rollout restart deployment/"${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}"
+      kubectl get pods -n "${APP_NAMESPACE}" -o wide
+
+  - type: Command
+    name: Show image pull events
+    command: |
+      sleep 30
+      kubectl get pods -n "${APP_NAMESPACE}" -o wide
+      kubectl get events -n "${APP_NAMESPACE}" --sort-by=.lastTimestamp | tail -40
--- a/k8s/broken/order-hpa-limited.yaml
+++ b/k8s/broken/order-hpa-limited.yaml
@@ -0,0 +1,22 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: order-service
+  namespace: kagent-demo
+  labels:
+    app: order-service
+    demo-scenario: hpa-limited-capacity
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: order-service
+  minReplicas: 2
+  maxReplicas: 3
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 60
--- a/k8s/broken/payment-error.yaml
+++ b/k8s/broken/payment-error.yaml
@@ -0,0 +1,41 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: payment-service
+  namespace: kagent-demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: payment-service
+  template:
+    metadata:
+      labels:
+        app: payment-service
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret-2
+      containers:
+        - name: payment-service
+          image: mty.ocir.io/xxxxxxxx/kagent-demo/payment-service:latest
+          ports:
+            - containerPort: 8082
+          env:
+            - name: DEMO_PAYMENT_FORCE_TIMEOUT
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_FORCE_TIMEOUT
+            - name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8082
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8082
--- a/k8s/broken/revenue-risk-configmap.yaml
+++ b/k8s/broken/revenue-risk-configmap.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: demo-config
+  namespace: kagent-demo
+data:
+  INVENTORY_FORCE_OUTAGE: "false"
+  PAYMENT_FORCE_TIMEOUT: "false"
+  PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999"
+  ORDER_STUCK_REVIEW_MODE: "false"
+  ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "30"
+  ORDER_REVENUE_AT_RISK_THRESHOLD: "5000"
+  INVENTORY_BASE_URL: "http://inventory-service:8081"
+  PAYMENT_BASE_URL: "http://payment-service:8082"
--- a/k8s/broken/simulate-three-errors-command-spec.yaml
+++ b/k8s/broken/simulate-three-errors-command-spec.yaml
@@ -0,0 +1,58 @@
+version: 0.1
+component: command
+timeoutInSeconds: 900
+shell: bash
+failImmediatelyOnError: true
+
+env:
+  variables:
+    APP_NAMESPACE: kagent-demo
+    PAYMENT_DEPLOYMENT: payment-service
+    PAYMENT_CONTAINER: payment-service
+    BROKEN_IMAGE_PULL_SECRET: missing-ocir-secret
+
+steps:
+  - type: Command
+    name: Apply business degradation ConfigMap
+    command: |
+      kubectl apply -f k8s/broken/revenue-risk-configmap.yaml
+      kubectl rollout restart deployment/order-service -n "${APP_NAMESPACE}"
+
+  - type: Command
+    name: Apply limited HPA
+    command: |
+      kubectl apply -f k8s/broken/order-hpa-limited.yaml
+      kubectl get hpa order-service -n "${APP_NAMESPACE}" -o wide
+
+  - type: Command
+    name: Break payment image pull secret
+    command: |
+      kubectl patch deployment "${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" \
+        --type merge \
+        -p "{
+          \"spec\": {
+            \"template\": {
+              \"spec\": {
+                \"imagePullSecrets\": [
+                  {\"name\": \"${BROKEN_IMAGE_PULL_SECRET}\"}
+                ],
+                \"containers\": [
+                  {
+                    \"name\": \"${PAYMENT_CONTAINER}\",
+                    \"imagePullPolicy\": \"Always\"
+                  }
+                ]
+              }
+            }
+          }
+        }"
+
+      kubectl rollout restart deployment/"${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}"
+
+  - type: Command
+    name: Show simulated failures
+    command: |
+      sleep 45
+      kubectl get pods -n "${APP_NAMESPACE}" -o wide
+      kubectl get hpa -n "${APP_NAMESPACE}" -o wide
+      kubectl get events -n "${APP_NAMESPACE}" --sort-by=.lastTimestamp | tail -60
--- a/k8s/delete-application-command-spec.yaml
+++ b/k8s/delete-application-command-spec.yaml
@@ -0,0 +1,145 @@
+version: 0.1
+component: command
+timeoutInSeconds: 900
+shell: bash
+failImmediatelyOnError: true
+
+env:
+  variables:
+    APP_NAMESPACE: kagent-demo
+    WAIT_TIMEOUT: 300s
+    OCI_REGION: mx-monterrey-1
+    OKE_CLUSTER_OCID: ocid1.cluster.oc1.mx-monterrey-1.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+    OKE_ENDPOINT_TYPE: PUBLIC_ENDPOINT
+
+steps:
+  - type: Command
+    name: Validate parameters
+    command: |
+      if [ -z "${APP_NAMESPACE}" ]; then
+        echo "APP_NAMESPACE parameter is required"
+        exit 1
+      fi
+
+      if [ -z "${WAIT_TIMEOUT}" ]; then
+        echo "WAIT_TIMEOUT parameter is required"
+        exit 1
+      fi
+
+      if [ -z "${OCI_REGION}" ]; then
+        echo "OCI_REGION parameter is required"
+        exit 1
+      fi
+
+      if [ -z "${OKE_CLUSTER_OCID}" ] || [ "${OKE_CLUSTER_OCID}" = "<oke-cluster-ocid>" ]; then
+        echo "OKE_CLUSTER_OCID parameter is required"
+        exit 1
+      fi
+
+      if [ -z "${OKE_ENDPOINT_TYPE}" ]; then
+        echo "OKE_ENDPOINT_TYPE parameter is required"
+        exit 1
+      fi
+
+      echo "APP_NAMESPACE=${APP_NAMESPACE}"
+      echo "WAIT_TIMEOUT=${WAIT_TIMEOUT}"
+      echo "OCI_REGION=${OCI_REGION}"
+      echo "OKE_CLUSTER_OCID=${OKE_CLUSTER_OCID}"
+      echo "OKE_ENDPOINT_TYPE=${OKE_ENDPOINT_TYPE}"
+
+  - type: Command
+    name: Configure kubeconfig
+    command: |
+      mkdir -p "${HOME}/.kube"
+
+      oci ce cluster create-kubeconfig \
+        --cluster-id "${OKE_CLUSTER_OCID}" \
+        --file "${HOME}/.kube/config" \
+        --region "${OCI_REGION}" \
+        --token-version 2.0.0 \
+        --kube-endpoint "${OKE_ENDPOINT_TYPE}"
+
+      export KUBECONFIG="${HOME}/.kube/config"
+      echo "KUBECONFIG=${KUBECONFIG}"
+
+      kubectl config current-context
+      kubectl cluster-info
+
+  - type: Command
+    name: Show cleanup target
+    command: |
+      echo "Deleting kagent demo application resources"
+      kubectl version --client=true
+      kubectl get namespace "${APP_NAMESPACE}"
+
+  - type: Command
+    name: Delete external entry point first
+    command: |
+      echo "Deleting LoadBalancer service first so OCI can release the public load balancer"
+      kubectl delete service order-service-lb \
+        -n "${APP_NAMESPACE}" \
+        --ignore-not-found=true
+
+  - type: Command
+    name: Delete application workloads
+    command: |
+      kubectl delete deployment order-service inventory-service payment-service \
+        -n "${APP_NAMESPACE}" \
+        --ignore-not-found=true
+
+      kubectl delete cronjob kagent-oke-remediator \
+        -n "${APP_NAMESPACE}" \
+        --ignore-not-found=true
+
+      kubectl delete job -l cronjob-name=kagent-oke-remediator \
+        -n "${APP_NAMESPACE}" \
+        --ignore-not-found=true
+
+      kubectl delete serviceaccount kagent-oke-remediator \
+        -n "${APP_NAMESPACE}" \
+        --ignore-not-found=true
+
+      kubectl wait --for=delete pod \
+        -l app=order-service \
+        -n "${APP_NAMESPACE}" \
+        --timeout="${WAIT_TIMEOUT}" || true
+
+      kubectl wait --for=delete pod \
+        -l app=inventory-service \
+        -n "${APP_NAMESPACE}" \
+        --timeout="${WAIT_TIMEOUT}" || true
+
+      kubectl wait --for=delete pod \
+        -l app=payment-service \
+        -n "${APP_NAMESPACE}" \
+        --timeout="${WAIT_TIMEOUT}" || true
+
+      kubectl wait --for=delete job \
+        -l cronjob-name=kagent-oke-remediator \
+        -n "${APP_NAMESPACE}" \
+        --timeout="${WAIT_TIMEOUT}" || true
+
+  - type: Command
+    name: Delete internal services and app config
+    command: |
+      kubectl delete service order-service inventory-service payment-service \
+        -n "${APP_NAMESPACE}" \
+        --ignore-not-found=true
+
+      kubectl delete configmap demo-config \
+        -n "${APP_NAMESPACE}" \
+        --ignore-not-found=true
+
+      kubectl delete hpa order-service inventory-service payment-service \
+        -n "${APP_NAMESPACE}" \
+        --ignore-not-found=true
+
+  - type: Command
+    name: Verify cleanup
+    command: |
+      echo "Remaining application resources in ${APP_NAMESPACE}:"
+      kubectl get deployment,svc,configmap,hpa,pod,cronjob,job,serviceaccount \
+        -n "${APP_NAMESPACE}" \
+        --ignore-not-found=true || true
+
+      echo "Application cleanup completed. Namespace and imagePullSecrets were preserved."
--- a/k8s/delete-namespace-command-spec.yaml
+++ b/k8s/delete-namespace-command-spec.yaml
@@ -0,0 +1,50 @@
+version: 0.1
+component: command
+timeoutInSeconds: 900
+shell: bash
+failImmediatelyOnError: true
+
+env:
+  variables:
+    APP_NAMESPACE: "kagent-demo"
+    WAIT_TIMEOUT: "300s"
+
+steps:
+  - type: Command
+    name: Confirm target namespace
+    command: |
+      echo "Full cleanup will delete the namespace and all resources inside it."
+      echo "APP_NAMESPACE=${APP_NAMESPACE}"
+
+      if [ "${APP_NAMESPACE}" = "default" ] || [ "${APP_NAMESPACE}" = "kube-system" ] || [ "${APP_NAMESPACE}" = "kagent" ]; then
+        echo "Refusing to delete protected namespace: ${APP_NAMESPACE}"
+        exit 1
+      fi
+
+      kubectl get namespace "${APP_NAMESPACE}"
+
+  - type: Command
+    name: Delete LoadBalancer before namespace cleanup
+    command: |
+      echo "Deleting LoadBalancer service first to start OCI load balancer cleanup"
+      kubectl delete service order-service-lb \
+        -n "${APP_NAMESPACE}" \
+        --ignore-not-found=true
+
+  - type: Command
+    name: Delete namespace
+    command: |
+      kubectl delete namespace "${APP_NAMESPACE}" \
+        --ignore-not-found=true \
+        --timeout="${WAIT_TIMEOUT}"
+
+  - type: Command
+    name: Verify namespace deletion
+    command: |
+      if kubectl get namespace "${APP_NAMESPACE}" >/dev/null 2>&1; then
+        echo "Namespace ${APP_NAMESPACE} still exists or is terminating."
+        kubectl get namespace "${APP_NAMESPACE}" -o wide
+        exit 1
+      fi
+
+      echo "Namespace ${APP_NAMESPACE} was deleted successfully."
--- a/k8s/fix-business-config-command-spec.yaml
+++ b/k8s/fix-business-config-command-spec.yaml
@@ -0,0 +1,87 @@
+version: 0.1
+component: command
+timeoutInSeconds: 900
+shell: bash
+failImmediatelyOnError: true
+
+steps:
+  - type: Command
+    name: Validate remediation parameters
+    command: |
+      required_vars=(
+        APP_NAMESPACE
+        CONFIGMAP_NAME
+        OCI_REGION
+        OKE_CLUSTER_OCID
+        OKE_ENDPOINT_TYPE
+        ORDER_MANUAL_REVIEW_RATE_THRESHOLD
+        ORDER_REVENUE_AT_RISK_THRESHOLD
+        ORDER_STUCK_REVIEW_MODE
+        PAYMENT_FORCE_TIMEOUT
+        INVENTORY_FORCE_OUTAGE
+        WAIT_TIMEOUT
+      )
+
+      for var_name in "${required_vars[@]}"; do
+        if [ -z "${!var_name}" ]; then
+          echo "Required parameter is missing: ${var_name}"
+          exit 1
+        fi
+      done
+
+      echo "APP_NAMESPACE=${APP_NAMESPACE}"
+      echo "CONFIGMAP_NAME=${CONFIGMAP_NAME}"
+      echo "OCI_REGION=${OCI_REGION}"
+      echo "OKE_CLUSTER_OCID=${OKE_CLUSTER_OCID}"
+      echo "OKE_ENDPOINT_TYPE=${OKE_ENDPOINT_TYPE}"
+      echo "ORDER_MANUAL_REVIEW_RATE_THRESHOLD=${ORDER_MANUAL_REVIEW_RATE_THRESHOLD}"
+      echo "ORDER_REVENUE_AT_RISK_THRESHOLD=${ORDER_REVENUE_AT_RISK_THRESHOLD}"
+      echo "ORDER_STUCK_REVIEW_MODE=${ORDER_STUCK_REVIEW_MODE}"
+      echo "PAYMENT_FORCE_TIMEOUT=${PAYMENT_FORCE_TIMEOUT}"
+      echo "INVENTORY_FORCE_OUTAGE=${INVENTORY_FORCE_OUTAGE}"
+      echo "WAIT_TIMEOUT=${WAIT_TIMEOUT}"
+
+  - type: Command
+    name: Configure kubeconfig
+    command: |
+      mkdir -p "${HOME}/.kube"
+
+      oci ce cluster create-kubeconfig \
+        --cluster-id "${OKE_CLUSTER_OCID}" \
+        --file "${HOME}/.kube/config" \
+        --region "${OCI_REGION}" \
+        --token-version 2.0.0 \
+        --kube-endpoint "${OKE_ENDPOINT_TYPE}"
+
+      export KUBECONFIG="${HOME}/.kube/config"
+      echo "KUBECONFIG=${KUBECONFIG}"
+
+      kubectl config current-context
+      kubectl cluster-info
+
+  - type: Command
+    name: Apply business ConfigMap remediation
+    command: |
+      kubectl patch configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" \
+        --type merge \
+        -p "{
+          \"data\": {
+            \"ORDER_MANUAL_REVIEW_RATE_THRESHOLD\": \"${ORDER_MANUAL_REVIEW_RATE_THRESHOLD}\",
+            \"ORDER_REVENUE_AT_RISK_THRESHOLD\": \"${ORDER_REVENUE_AT_RISK_THRESHOLD}\",
+            \"ORDER_STUCK_REVIEW_MODE\": \"${ORDER_STUCK_REVIEW_MODE}\",
+            \"PAYMENT_FORCE_TIMEOUT\": \"${PAYMENT_FORCE_TIMEOUT}\",
+            \"INVENTORY_FORCE_OUTAGE\": \"${INVENTORY_FORCE_OUTAGE}\"
+          }
+        }"
+
+  - type: Command
+    name: Restart order service
+    command: |
+      kubectl rollout restart deployment/order-service -n "${APP_NAMESPACE}"
+      kubectl rollout status deployment/order-service -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}"
+
+  - type: Command
+    name: Show remediated configuration
+    command: |
+      kubectl get configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" -o yaml
+      kubectl get pods -n "${APP_NAMESPACE}" -o wide
--- a/k8s/fix-imagepull-secret-command-spec.yaml
+++ b/k8s/fix-imagepull-secret-command-spec.yaml
@@ -0,0 +1,66 @@
+version: 0.1
+component: command
+timeoutInSeconds: 900
+shell: bash
+failImmediatelyOnError: true
+
+env:
+  variables:
+    APP_NAMESPACE: kagent-demo
+    DEPLOYMENT_NAME: payment-service
+    CONTAINER_NAME: payment-service
+    IMAGE_PULL_SECRET: ocir-secret
+    WAIT_TIMEOUT: 300s
+
+steps:
+  - type: Command
+    name: Restore valid imagePullSecret
+    command: |
+      case "${DEPLOYMENT_NAME}" in
+        order-service|payment-service|inventory-service)
+          echo "Target deployment is valid: ${DEPLOYMENT_NAME}"
+          ;;
+        *)
+          echo "Invalid DEPLOYMENT_NAME=${DEPLOYMENT_NAME}"
+          echo "Allowed values: order-service, payment-service, inventory-service"
+          exit 1
+          ;;
+      esac
+
+      if [ -z "${CONTAINER_NAME}" ]; then
+        CONTAINER_NAME="${DEPLOYMENT_NAME}"
+      fi
+
+      echo "APP_NAMESPACE=${APP_NAMESPACE}"
+      echo "DEPLOYMENT_NAME=${DEPLOYMENT_NAME}"
+      echo "CONTAINER_NAME=${CONTAINER_NAME}"
+      echo "IMAGE_PULL_SECRET=${IMAGE_PULL_SECRET}"
+
+      kubectl patch deployment "${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" \
+        --type merge \
+        -p "{
+          \"spec\": {
+            \"template\": {
+              \"spec\": {
+                \"imagePullSecrets\": [
+                  {\"name\": \"${IMAGE_PULL_SECRET}\"}
+                ],
+                \"containers\": [
+                  {
+                    \"name\": \"${CONTAINER_NAME}\",
+                    \"imagePullPolicy\": \"IfNotPresent\"
+                  }
+                ]
+              }
+            }
+          }
+        }"
+
+      kubectl rollout restart deployment/"${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}"
+      kubectl rollout status deployment/"${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}"
+
+  - type: Command
+    name: Verify restored image pull
+    command: |
+      kubectl get deployment "${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" -o jsonpath='{.spec.template.spec.imagePullSecrets}{"\n"}'
+      kubectl get pods -n "${APP_NAMESPACE}" -o wide
--- a/k8s/fix-order-hpa-command-spec.yaml
+++ b/k8s/fix-order-hpa-command-spec.yaml
@@ -0,0 +1,44 @@
+version: 0.1
+component: command
+timeoutInSeconds: 600
+shell: bash
+failImmediatelyOnError: true
+
+env:
+  variables:
+    APP_NAMESPACE: kagent-demo
+    HPA_NAME: order-service
+    MIN_REPLICAS: "3"
+    MAX_REPLICAS: "8"
+    CPU_TARGET: "65"
+
+steps:
+  - type: Command
+    name: Patch order HPA capacity
+    command: |
+      kubectl patch hpa "${HPA_NAME}" -n "${APP_NAMESPACE}" \
+        --type merge \
+        -p "{
+          \"spec\": {
+            \"minReplicas\": ${MIN_REPLICAS},
+            \"maxReplicas\": ${MAX_REPLICAS},
+            \"metrics\": [
+              {
+                \"type\": \"Resource\",
+                \"resource\": {
+                  \"name\": \"cpu\",
+                  \"target\": {
+                    \"type\": \"Utilization\",
+                    \"averageUtilization\": ${CPU_TARGET}
+                  }
+                }
+              }
+            ]
+          }
+        }"
+
+  - type: Command
+    name: Show HPA state
+    command: |
+      kubectl get hpa "${HPA_NAME}" -n "${APP_NAMESPACE}" -o wide
+      kubectl describe hpa "${HPA_NAME}" -n "${APP_NAMESPACE}"
--- a/k8s/kagent-multi-error-payload.json
+++ b/k8s/kagent-multi-error-payload.json
@@ -0,0 +1,25 @@
+{
+  "responseMode": "json",
+  "region": "${REGION}",
+  "appNamespace": "${APP_NAMESPACE}",
+  "observabilityUrl": "${ORDER_OBSERVABILITY_URL}",
+  "notificationTopicOcid": "${NOTIFICATION_TOPIC_OCID}",
+  "remediationPipelines": {
+    "business": {
+      "approvalUrl": "${BUSINESS_REMEDIATION_APPROVAL_URL}",
+      "projectOcid": "${BUSINESS_REMEDIATION_PROJECT_OCID}",
+      "pipelineOcid": "${BUSINESS_REMEDIATION_PIPELINE_OCID}"
+    },
+    "imagePull": {
+      "approvalUrl": "${IMAGEPULL_REMEDIATION_APPROVAL_URL}",
+      "projectOcid": "${IMAGEPULL_REMEDIATION_PROJECT_OCID}",
+      "pipelineOcid": "${IMAGEPULL_REMEDIATION_PIPELINE_OCID}"
+    },
+    "hpa": {
+      "approvalUrl": "${HPA_REMEDIATION_APPROVAL_URL}",
+      "projectOcid": "${HPA_REMEDIATION_PROJECT_OCID}",
+      "pipelineOcid": "${HPA_REMEDIATION_PIPELINE_OCID}"
+    }
+  },
+  "prompt": "Analiza el namespace kagent-demo despues del despliegue OCI DevOps. La URL de observabilidad obligatoria esta en el campo observabilityUrl del payload. Debes consultarla antes de responder y no debes preguntar por esa URL. Busca fallas activas, no eventos historicos, en estas categorias: BUSINESS_PROCESS_DEGRADATION, IMAGE_PULL_FAILURE, CRASH_LOOP_FAILURE, PROBE_FAILURE, HPA_AUTOSCALING_FAILURE, POD_SCHEDULING_FAILURE, DEPENDENCY_FAILURE, OKE_PLATFORM_FAILURE y NO_ACTIVE_FAILURE. Revisa pods, deployments, services, endpoints, eventos recientes, ConfigMap demo-config, imagePullSecrets, service accounts, HPA, logs recientes de order-service, inventory-service y payment-service, y el dashboard/API de observabilidad. Para degradacion de negocio, usa la respuesta del dashboard/API como fuente de verdad: si businessHealth.healthy=false, services incluye order-service con status DEGRADED, manualReviewRate supera el umbral configurado, pendingReviewOrders es alto o revenueAtRisk supera ORDER_REVENUE_AT_RISK_THRESHOLD, debes clasificar BUSINESS_PROCESS_DEGRADATION aunque todos los pods esten Running. Detecta si existen uno o varios problemas al mismo tiempo. Para cada problema, entrega evidencia concreta, causa raiz probable, impacto y accion recomendada para OCI DevOps. Si hay degradacion de negocio por cardToken con prefijo 9999, explica que PAYMENT_MANUAL_REVIEW_CARD_PREFIX envia esas ordenes a PENDING_REVIEW y que ORDER_MANUAL_REVIEW_RATE_THRESHOLD y ORDER_REVENUE_AT_RISK_THRESHOLD definen cuando order-service pasa a DEGRADED. Si hay degradacion de negocio, recomienda valores corregidos del ConfigMap. Si hay ImagePullBackOff o ErrImagePull, identifica deployment, imagen afectada y secreto faltante o invalido. Si hay HPA limitado, identifica si desired replicas alcanzo maxReplicas y recomienda nuevo minReplicas, maxReplicas y target CPU. Incluye un campo report para correo con title, executiveSummary, rootCause, businessImpact, evidence, configMapAnalysis y recommendedAction. El report debe ser entendible por un aprobador de OCI DevOps y debe explicar claramente por que la aplicacion esta degradada y que pipeline de remediacion debe aprobar. Responde exclusivamente JSON puro, sin markdown, sin texto antes ni despues, con este formato: {\"status\":\"HEALTHY|DEGRADED|FAILED\",\"summary\":\"\",\"report\":{\"title\":\"\",\"executiveSummary\":\"\",\"rootCause\":\"\",\"businessImpact\":\"\",\"evidence\":[],\"configMapAnalysis\":\"\",\"recommendedAction\":\"\"},\"problems\":[{\"failureType\":\"BUSINESS_PROCESS_DEGRADATION|IMAGE_PULL_FAILURE|HPA_AUTOSCALING_FAILURE|NO_ACTIVE_FAILURE\",\"rootCause\":\"\",\"evidence\":[],\"impact\":\"\",\"recommendedAction\":\"CONTINUE|APPROVAL_REQUIRED|APPLY_CONFIGMAP_FIX|FIX_IMAGE_PULL_SECRET|PATCH_HPA|ROLLBACK|REDEPLOY\",\"affectedResources\":[],\"recommendedConfigMapData\":{},\"recommendedHpaPatch\":{},\"deploymentsToRestart\":[]}],\"overallRecommendedAction\":\"CONTINUE|APPROVAL_REQUIRED|APPLY_CONFIGMAP_FIX|FIX_IMAGE_PULL_SECRET|PATCH_HPA|ROLLBACK|REDEPLOY\"}"
+}
--- a/k8s/kagent/environment_variables.sh
+++ b/k8s/kagent/environment_variables.sh
@@ -0,0 +1,6 @@
+export OCI_GENAI_API_KEY="sk-eavZt2RSpxxxxxxxxxxxxxxxxxxxxxxxxx"
+export OCI_GENAI_REGION="us-chicago-1"
+export OCI_GENAI_BASE_URL="https://inference.generativeai.${OCI_GENAI_REGION}.oci.oraclecloud.com/openai/v1"
+export OCI_GENAI_MODEL="openai.gpt-4o"
+export OCI_GENAI_PROJECT_OCID="ocid1.generativeaiproject.oc1.us-chicago-1.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
--- a/k8s/kagent/kagent-mcp-lb.yaml
+++ b/k8s/kagent/kagent-mcp-lb.yaml
@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kagent-controller-lb
+  namespace: kagent
+  annotations:
+    oci.oraclecloud.com/load-balancer-type: "nlb"
+    oci.oraclecloud.com/reserved-ips: "40.233.18.237"
+  labels:
+    app.kubernetes.io/instance: kagent
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kagent
+    app.kubernetes.io/part-of: kagent
+    app.kubernetes.io/version: 0.9.12
+    helm.sh/chart: kagent-0.9.12
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+  ports:
+    - name: mcp
+      port: 8083
+      protocol: TCP
+      targetPort: 8083
+      nodePort: 31083
+  selector:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/instance: kagent
+    app.kubernetes.io/name: kagent
--- a/k8s/kagent/kagent-ui-lb-exported.yaml
+++ b/k8s/kagent/kagent-ui-lb-exported.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kagent-ui-lb
+  namespace: kagent
+  annotations:
+    oci.oraclecloud.com/load-balancer-type: "nlb"
+    oci.oraclecloud.com/reserved-ips: "40.233.25.148"
+  labels:
+    app.kubernetes.io/instance: kagent
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: kagent
+    app.kubernetes.io/part-of: kagent
+    app.kubernetes.io/version: 0.9.12
+    helm.sh/chart: kagent-0.9.12
+spec:
+  type: LoadBalancer
+  externalTrafficPolicy: Cluster
+  ports:
+    - port: 80
+      protocol: TCP
+      targetPort: 8080
+      nodePort: 31002
+  selector:
+    app.kubernetes.io/component: ui
+    app.kubernetes.io/instance: kagent
+    app.kubernetes.io/name: kagent
--- a/k8s/oke-remediator-multi-check/README.md
+++ b/k8s/oke-remediator-multi-check/README.md
@@ -0,0 +1,24 @@
+# OKE Remediator Multi-Check
+
+This workload runs every 3 minutes and performs three independent kagent checks as separate MCP interactions:
+
+1. Business application health for Orders
+2. HPA autoscaling analysis for `order-service`
+3. ImagePull analysis for application deployments
+
+Each result is normalized to a structured JSON shape. The worker stores one state entry per `error_code` in a Kubernetes ConfigMap and only sends one email plus one remediation deployment trigger per active incident signature.
+
+## Deploy
+
+```bash
+kubectl apply -f k8s/oke-remediator-multi-check/serviceaccount.yaml
+kubectl apply -f k8s/oke-remediator-multi-check/role.yaml
+kubectl apply -f k8s/oke-remediator-multi-check/rolebinding.yaml
+kubectl apply -f k8s/oke-remediator-multi-check/cronjob.yaml
+```
+
+## Notes
+
+- The remediation pipelines must include approval stages.
+- The notification email includes the remediation deployment approval link when the deployment is created successfully.
+- The worker is intended for OKE Workload Identity.
--- a/k8s/oke-remediator-multi-check/cronjob.yaml
+++ b/k8s/oke-remediator-multi-check/cronjob.yaml
@@ -0,0 +1,50 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: kagent-oke-remediator-multi
+  namespace: kagent-demo
+spec:
+  schedule: "*/3 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        spec:
+          restartPolicy: Never
+          serviceAccountName: kagent-oke-remediator-multi
+          imagePullSecrets:
+            - name: ocir-secret
+          containers:
+            - name: kagent-oke-remediator-multi
+              image: ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-multi-check:${IMAGE_TAG}
+              imagePullPolicy: Always
+              env:
+                - name: OCI_REGION
+                  value: mx-monterrey-1
+                - name: APP_NAMESPACE
+                  value: kagent-demo
+                - name: KAGENT_MCP_URL
+                  value: http://kagent-controller.kagent.svc.cluster.local:8083/mcp
+                - name: KAGENT_AGENT_NAME
+                  value: k8s-agent
+                - name: KAGENT_AGENT_NAMESPACE
+                  value: kagent
+                - name: ORDER_OBSERVABILITY_URL
+                  value: http://order-service-lb.kagent-demo.svc.cluster.local/api/observability
+                - name: NOTIFICATION_TOPIC_OCID
+                  value: ${NOTIFICATION_TOPIC_OCID}
+                - name: BUSINESS_REMEDIATION_PROJECT_OCID
+                  value: ${BUSINESS_REMEDIATION_PROJECT_OCID}
+                - name: BUSINESS_REMEDIATION_PIPELINE_OCID
+                  value: ${BUSINESS_REMEDIATION_PIPELINE_OCID}
+                - name: IMAGEPULL_REMEDIATION_PROJECT_OCID
+                  value: ${IMAGEPULL_REMEDIATION_PROJECT_OCID}
+                - name: IMAGEPULL_REMEDIATION_PIPELINE_OCID
+                  value: ${IMAGEPULL_REMEDIATION_PIPELINE_OCID}
+                - name: HPA_REMEDIATION_PROJECT_OCID
+                  value: ${HPA_REMEDIATION_PROJECT_OCID}
+                - name: HPA_REMEDIATION_PIPELINE_OCID
+                  value: ${HPA_REMEDIATION_PIPELINE_OCID}
--- a/k8s/oke-remediator-multi-check/role.yaml
+++ b/k8s/oke-remediator-multi-check/role.yaml
@@ -0,0 +1,30 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kagent-oke-remediator-multi
+  namespace: kagent-demo
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - events
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: ["apps"]
+    resources:
+      - deployments
+      - replicasets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - create
+      - update
+      - patch
--- a/k8s/oke-remediator-multi-check/rolebinding.yaml
+++ b/k8s/oke-remediator-multi-check/rolebinding.yaml
@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kagent-oke-remediator-multi
+  namespace: kagent-demo
+subjects:
+  - kind: ServiceAccount
+    name: kagent-oke-remediator-multi
+    namespace: kagent-demo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kagent-oke-remediator-multi
--- a/k8s/oke-remediator-multi-check/serviceaccount.yaml
+++ b/k8s/oke-remediator-multi-check/serviceaccount.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kagent-oke-remediator-multi
+  namespace: kagent-demo
--- a/k8s/oke-remediator/README.md
+++ b/k8s/oke-remediator/README.md
@@ -0,0 +1,50 @@
+# OKE Remediator Job
+
+This workload runs inside OKE as a `CronJob` and performs the same high-level flow as the OCI Function:
+
+1. Queries the business observability API.
+2. Sends a task to `kagent` through MCP.
+3. Detects business degradation, image pull failures, or HPA capacity issues.
+4. Triggers the corresponding OCI DevOps remediation deployment for approval.
+5. Sends an OCI Notifications email with the diagnostic report and the approval link to the remediation deployment.
+
+## Authentication
+
+The workload is configured to use OKE Workload Identity first, and falls back to generic OCI resource principals only if workload identity is not available.
+
+For OKE Workload Identity, the important pieces are:
+
+- Enhanced OKE cluster.
+- Kubernetes `ServiceAccount`.
+- IAM policy scoped to:
+  - `request.principal.type = 'workload'`
+  - `request.principal.namespace = 'kagent-demo'`
+  - `request.principal.service_account = 'kagent-oke-remediator'`
+  - `request.principal.cluster_id = '<cluster-ocid>'`
+
+No OCI config file mount is required for the manifest included here.
+
+## Deployment
+
+Apply:
+
+```bash
+kubectl apply -f k8s/oke-remediator/serviceaccount.yaml
+kubectl apply -f k8s/oke-remediator/role.yaml
+kubectl apply -f k8s/oke-remediator/rolebinding.yaml
+kubectl apply -f k8s/oke-remediator/cronjob.yaml
+```
+
+Run one job immediately for validation:
+
+```bash
+kubectl create job --from=cronjob/kagent-oke-remediator kagent-oke-remediator-manual -n kagent-demo
+kubectl logs job/kagent-oke-remediator-manual -n kagent-demo -f
+```
+
+## Notes
+
+- The remediation pipelines must include an approval stage.
+- The job triggers the remediation deployment and sends the email with the deployment execution link.
+- The job stores the last active incident signature in the `kagent-oke-remediator-state` ConfigMap to suppress duplicate emails for the same active issue.
+- The Python worker uses `oci.auth.signers.get_oke_workload_identity_resource_principal_signer()` before any fallback signer.
--- a/k8s/oke-remediator/cronjob.yaml
+++ b/k8s/oke-remediator/cronjob.yaml
@@ -0,0 +1,56 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: kagent-oke-remediator
+  namespace: kagent-demo
+spec:
+  schedule: "*/3 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        spec:
+          restartPolicy: Never
+          serviceAccountName: kagent-oke-remediator
+          imagePullSecrets:
+            - name: ocir-secret
+          containers:
+            - name: kagent-oke-remediator
+              image: ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-job:${IMAGE_TAG}
+              imagePullPolicy: Always
+              env:
+                - name: OCI_REGION
+                  value: mx-monterrey-1
+                - name: APP_NAMESPACE
+                  value: kagent-demo
+                - name: KAGENT_MCP_URL
+                  value: http://kagent-controller.kagent.svc.cluster.local:8083/mcp
+                - name: KAGENT_AGENT_NAME
+                  value: k8s-agent
+                - name: KAGENT_AGENT_NAMESPACE
+                  value: kagent
+                - name: ORDER_OBSERVABILITY_URL
+                  value: http://order-service:8080/api/observability
+                - name: NOTIFICATION_TOPIC_OCID
+                  value: ${NOTIFICATION_TOPIC_OCID}
+                - name: BUSINESS_REMEDIATION_PIPELINE_URL
+                  value: ${BUSINESS_REMEDIATION_PIPELINE_URL}
+                - name: BUSINESS_REMEDIATION_PROJECT_OCID
+                  value: ${BUSINESS_REMEDIATION_PROJECT_OCID}
+                - name: BUSINESS_REMEDIATION_PIPELINE_OCID
+                  value: ${BUSINESS_REMEDIATION_PIPELINE_OCID}
+                - name: IMAGEPULL_REMEDIATION_PIPELINE_URL
+                  value: ${IMAGEPULL_REMEDIATION_PIPELINE_URL}
+                - name: IMAGEPULL_REMEDIATION_PROJECT_OCID
+                  value: ${IMAGEPULL_REMEDIATION_PROJECT_OCID}
+                - name: IMAGEPULL_REMEDIATION_PIPELINE_OCID
+                  value: ${IMAGEPULL_REMEDIATION_PIPELINE_OCID}
+                - name: HPA_REMEDIATION_PIPELINE_URL
+                  value: ${HPA_REMEDIATION_PIPELINE_URL}
+                - name: HPA_REMEDIATION_PROJECT_OCID
+                  value: ${HPA_REMEDIATION_PROJECT_OCID}
+                - name: HPA_REMEDIATION_PIPELINE_OCID
+                  value: ${HPA_REMEDIATION_PIPELINE_OCID}
--- a/k8s/oke-remediator/role.yaml
+++ b/k8s/oke-remediator/role.yaml
@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kagent-oke-remediator
+  namespace: kagent-demo
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "create", "update", "patch"]
--- a/k8s/oke-remediator/rolebinding.yaml
+++ b/k8s/oke-remediator/rolebinding.yaml
@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kagent-oke-remediator
+  namespace: kagent-demo
+subjects:
+  - kind: ServiceAccount
+    name: kagent-oke-remediator
+    namespace: kagent-demo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kagent-oke-remediator
--- a/k8s/oke-remediator/serviceaccount.yaml
+++ b/k8s/oke-remediator/serviceaccount.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kagent-oke-remediator
+  namespace: kagent-demo
--- a/k8s/remediate-three-errors-command-spec.yaml
+++ b/k8s/remediate-three-errors-command-spec.yaml
@@ -0,0 +1,94 @@
+version: 0.1
+component: command
+timeoutInSeconds: 1200
+shell: bash
+failImmediatelyOnError: true
+
+env:
+  variables:
+    APP_NAMESPACE: kagent-demo
+    CONFIGMAP_NAME: demo-config
+    PAYMENT_DEPLOYMENT: payment-service
+    PAYMENT_CONTAINER: payment-service
+    IMAGE_PULL_SECRET: ocir-secret
+    ORDER_HPA_NAME: order-service
+    ORDER_MIN_REPLICAS: "3"
+    ORDER_MAX_REPLICAS: "8"
+    ORDER_CPU_TARGET: "65"
+    WAIT_TIMEOUT: 300s
+
+steps:
+  - type: Command
+    name: Fix business ConfigMap
+    command: |
+      kubectl patch configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" \
+        --type merge \
+        -p '{
+          "data": {
+            "ORDER_STUCK_REVIEW_MODE": "false",
+            "ORDER_MANUAL_REVIEW_RATE_THRESHOLD": "70",
+            "ORDER_REVENUE_AT_RISK_THRESHOLD": "15000",
+            "PAYMENT_FORCE_TIMEOUT": "false",
+            "INVENTORY_FORCE_OUTAGE": "false"
+          }
+        }'
+
+      kubectl rollout restart deployment/order-service -n "${APP_NAMESPACE}"
+
+  - type: Command
+    name: Fix payment image pull secret
+    command: |
+      kubectl patch deployment "${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" \
+        --type merge \
+        -p "{
+          \"spec\": {
+            \"template\": {
+              \"spec\": {
+                \"imagePullSecrets\": [
+                  {\"name\": \"${IMAGE_PULL_SECRET}\"}
+                ],
+                \"containers\": [
+                  {
+                    \"name\": \"${PAYMENT_CONTAINER}\",
+                    \"imagePullPolicy\": \"IfNotPresent\"
+                  }
+                ]
+              }
+            }
+          }
+        }"
+
+      kubectl rollout restart deployment/"${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}"
+
+  - type: Command
+    name: Fix order HPA ceiling
+    command: |
+      kubectl patch hpa "${ORDER_HPA_NAME}" -n "${APP_NAMESPACE}" \
+        --type merge \
+        -p "{
+          \"spec\": {
+            \"minReplicas\": ${ORDER_MIN_REPLICAS},
+            \"maxReplicas\": ${ORDER_MAX_REPLICAS},
+            \"metrics\": [
+              {
+                \"type\": \"Resource\",
+                \"resource\": {
+                  \"name\": \"cpu\",
+                  \"target\": {
+                    \"type\": \"Utilization\",
+                    \"averageUtilization\": ${ORDER_CPU_TARGET}
+                  }
+                }
+              }
+            ]
+          }
+        }"
+
+  - type: Command
+    name: Wait and verify remediation
+    command: |
+      kubectl rollout status deployment/order-service -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}"
+      kubectl rollout status deployment/"${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}"
+      kubectl get pods -n "${APP_NAMESPACE}" -o wide
+      kubectl get hpa -n "${APP_NAMESPACE}" -o wide
+      kubectl get configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" -o yaml
--- a/k8s/remediation/README.md
+++ b/k8s/remediation/README.md
@@ -0,0 +1,71 @@
+# Declarative Remediation Manifests
+
+Use these manifests with OCI DevOps `Apply manifest to your Kubernetes cluster` stages.
+
+## Business remediation
+
+Apply in this order:
+
+```text
+business-configmap.yaml
+order-rollout-restart.yaml
+```
+
+`business-configmap.yaml` sets safe business thresholds.
+
+`order-rollout-restart.yaml` reapplies the `order-service` deployment template so pods reload ConfigMap-backed environment variables.
+
+Before applying `order-rollout-restart.yaml`, replace:
+
+```text
+<restart-token>
+```
+
+with a new value, for example the OCI DevOps deployment ID, build number, or timestamp. If the annotation does not change, Kubernetes will not restart the pods.
+
+## ImagePull remediation
+
+Use the manifest matching the affected deployment:
+
+```text
+imagepull-all-services.yaml
+imagepull-order.yaml
+imagepull-payment.yaml
+imagepull-inventory.yaml
+```
+
+For the demo, prefer `imagepull-all-services.yaml`. It restores `ocir-secret` across all three business deployments, so the same remediation pipeline works regardless of which service hit `ImagePullBackOff`.
+
+Each manifest restores:
+
+```yaml
+imagePullSecrets:
+  - name: ocir-secret
+```
+
+Before applying an ImagePull remediation manifest, replace:
+
+```text
+<region-key>
+<tenancy-namespace>
+<image-tag>
+<restart-token>
+```
+
+with the same values used by the active deployment. The `restart-token` forces a new ReplicaSet after the secret is restored.
+
+## HPA remediation
+
+Use:
+
+```text
+order-hpa-capacity.yaml
+```
+
+It changes `order-service` HPA to:
+
+```text
+minReplicas=3
+maxReplicas=8
+averageUtilization=65
+```
--- a/k8s/remediation/business-configmap.yaml
+++ b/k8s/remediation/business-configmap.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: demo-config
+  namespace: kagent-demo
+data:
+  INVENTORY_FORCE_OUTAGE: "false"
+  PAYMENT_FORCE_TIMEOUT: "false"
+  PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999"
+  ORDER_STUCK_REVIEW_MODE: "false"
+  ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "70"
+  ORDER_REVENUE_AT_RISK_THRESHOLD: "15000"
+  INVENTORY_BASE_URL: "http://inventory-service:8081"
+  PAYMENT_BASE_URL: "http://payment-service:8082"
--- a/k8s/remediation/imagepull-all-services.yaml
+++ b/k8s/remediation/imagepull-all-services.yaml
@@ -0,0 +1,189 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: order-service
+  namespace: kagent-demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: order-service
+  template:
+    metadata:
+      labels:
+        app: order-service
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret
+      containers:
+        - name: order-service
+          image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/order-service:<image-tag>
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: 1000m
+              memory: 1024Mi
+          env:
+            - name: SERVICES_INVENTORY_BASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: INVENTORY_BASE_URL
+            - name: SERVICES_PAYMENT_BASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_BASE_URL
+            - name: DEMO_ORDERS_STUCK_REVIEW_MODE
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_STUCK_REVIEW_MODE
+            - name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
+            - name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_REVENUE_AT_RISK_THRESHOLD
+          startupProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8080
+            failureThreshold: 30
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8080
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 3
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: payment-service
+  namespace: kagent-demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: payment-service
+  template:
+    metadata:
+      labels:
+        app: payment-service
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret
+      containers:
+        - name: payment-service
+          image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/payment-service:<image-tag>
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8082
+          env:
+            - name: DEMO_PAYMENT_FORCE_TIMEOUT
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_FORCE_TIMEOUT
+            - name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX
+          startupProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8082
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 18
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8082
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8082
+            initialDelaySeconds: 60
+            periodSeconds: 20
+            timeoutSeconds: 5
+            failureThreshold: 6
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: inventory-service
+  namespace: kagent-demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: inventory-service
+  template:
+    metadata:
+      labels:
+        app: inventory-service
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret
+      containers:
+        - name: inventory-service
+          image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/inventory-service:<image-tag>
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8081
+          env:
+            - name: DEMO_INVENTORY_FORCE_OUTAGE
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: INVENTORY_FORCE_OUTAGE
+          startupProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8081
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 18
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8081
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8081
+            initialDelaySeconds: 60
+            periodSeconds: 20
+            timeoutSeconds: 5
+            failureThreshold: 6
--- a/k8s/remediation/imagepull-inventory.yaml
+++ b/k8s/remediation/imagepull-inventory.yaml
@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: inventory-service
+  namespace: kagent-demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: inventory-service
+  template:
+    metadata:
+      labels:
+        app: inventory-service
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret
+      containers:
+        - name: inventory-service
+          image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/inventory-service:<image-tag>
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8081
+          env:
+            - name: DEMO_INVENTORY_FORCE_OUTAGE
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: INVENTORY_FORCE_OUTAGE
+          startupProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8081
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 18
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8081
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8081
+            initialDelaySeconds: 60
+            periodSeconds: 20
+            timeoutSeconds: 5
+            failureThreshold: 6
--- a/k8s/remediation/imagepull-order.yaml
+++ b/k8s/remediation/imagepull-order.yaml
@@ -0,0 +1,76 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: order-service
+  namespace: kagent-demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: order-service
+  template:
+    metadata:
+      labels:
+        app: order-service
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret
+      containers:
+        - name: order-service
+          image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/order-service:<image-tag>
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8080
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: 1000m
+              memory: 1024Mi
+          env:
+            - name: SERVICES_INVENTORY_BASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: INVENTORY_BASE_URL
+            - name: SERVICES_PAYMENT_BASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_BASE_URL
+            - name: DEMO_ORDERS_STUCK_REVIEW_MODE
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_STUCK_REVIEW_MODE
+            - name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
+            - name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_REVENUE_AT_RISK_THRESHOLD
+          startupProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8080
+            failureThreshold: 30
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8080
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 3
--- a/k8s/remediation/imagepull-payment.yaml
+++ b/k8s/remediation/imagepull-payment.yaml
@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: payment-service
+  namespace: kagent-demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: payment-service
+  template:
+    metadata:
+      labels:
+        app: payment-service
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret
+      containers:
+        - name: payment-service
+          image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/payment-service:<image-tag>
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 8082
+          env:
+            - name: DEMO_PAYMENT_FORCE_TIMEOUT
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_FORCE_TIMEOUT
+            - name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX
+          startupProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8082
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 18
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8082
+            initialDelaySeconds: 20
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8082
+            initialDelaySeconds: 60
+            periodSeconds: 20
+            timeoutSeconds: 5
+            failureThreshold: 6
--- a/k8s/remediation/order-hpa-capacity.yaml
+++ b/k8s/remediation/order-hpa-capacity.yaml
@@ -0,0 +1,22 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: order-service
+  namespace: kagent-demo
+  labels:
+    app: order-service
+    demo.oracle.com/remediation: hpa-capacity
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: order-service
+  minReplicas: 3
+  maxReplicas: 8
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 65
--- a/k8s/remediation/order-rollout-restart.yaml
+++ b/k8s/remediation/order-rollout-restart.yaml
@@ -0,0 +1,77 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: order-service
+  namespace: kagent-demo
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: order-service
+  template:
+    metadata:
+      labels:
+        app: order-service
+      annotations:
+        demo.oracle.com/restart-token: "<restart-token>"
+    spec:
+      imagePullSecrets:
+        - name: ocir-secret
+      containers:
+        - name: order-service
+          image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/order-service:latest
+          ports:
+            - containerPort: 8080
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: 1000m
+              memory: 1024Mi
+          env:
+            - name: SERVICES_INVENTORY_BASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: INVENTORY_BASE_URL
+            - name: SERVICES_PAYMENT_BASE_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: PAYMENT_BASE_URL
+            - name: DEMO_ORDERS_STUCK_REVIEW_MODE
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_STUCK_REVIEW_MODE
+            - name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
+            - name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
+              valueFrom:
+                configMapKeyRef:
+                  name: demo-config
+                  key: ORDER_REVENUE_AT_RISK_THRESHOLD
+          startupProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8080
+            failureThreshold: 30
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /actuator/health/readiness
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            failureThreshold: 6
+          livenessProbe:
+            httpGet:
+              path: /actuator/health/liveness
+              port: 8080
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            failureThreshold: 3
--- a/oke-remediator-multi-check/main.py
+++ b/oke-remediator-multi-check/main.py
--- a/oke-remediator-multi-check/requirements.txt
+++ b/oke-remediator-multi-check/requirements.txt
@@ -0,0 +1,2 @@
+mcp>=1.13.0,<3
+oci>=2.129.0
--- a/order-service/pom.xml
+++ b/order-service/pom.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.oracle.demo</groupId>
+        <artifactId>kagent-oci-devops-demo</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>order-service</artifactId>
+    <name>order-service</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-actuator</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-validation</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-prometheus</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>
--- a/order-service/src/main/java/com/oracle/demo/orders/OrderServiceApplication.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/OrderServiceApplication.java
@@ -0,0 +1,19 @@
+package com.oracle.demo.orders;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.web.client.RestClient;
+import org.springframework.context.annotation.Bean;
+
+@SpringBootApplication
+public class OrderServiceApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(OrderServiceApplication.class, args);
+    }
+
+    @Bean
+    RestClient restClient() {
+        return RestClient.builder().build();
+    }
+}
--- a/order-service/src/main/java/com/oracle/demo/orders/api/OrderController.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/api/OrderController.java
@@ -0,0 +1,41 @@
+package com.oracle.demo.orders.api;
+
+import com.oracle.demo.orders.model.BusinessHealth;
+import com.oracle.demo.orders.model.CreateOrderRequest;
+import com.oracle.demo.orders.model.OrderRecord;
+import com.oracle.demo.orders.service.OrderService;
+import jakarta.validation.Valid;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.List;
+
+@RestController
+@RequestMapping("/api/orders")
+public class OrderController {
+
+    private final OrderService orderService;
+
+    public OrderController(OrderService orderService) {
+        this.orderService = orderService;
+    }
+
+    @PostMapping
+    public OrderRecord createOrder(@Valid @RequestBody CreateOrderRequest request) {
+        return orderService.createOrder(request);
+    }
+
+    @GetMapping
+    public List<OrderRecord> listOrders() {
+        return orderService.listOrders();
+    }
+
+    @GetMapping("/business-health")
+    public BusinessHealth businessHealth() {
+        return orderService.businessHealth();
+    }
+}
+
--- a/order-service/src/main/java/com/oracle/demo/orders/client/InventoryClient.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/client/InventoryClient.java
@@ -0,0 +1,26 @@
+package com.oracle.demo.orders.client;
+
+import com.oracle.demo.orders.model.InventoryReservation;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.client.RestClient;
+
+@Component
+public class InventoryClient {
+
+    private final RestClient restClient;
+    private final String inventoryBaseUrl;
+
+    public InventoryClient(RestClient restClient, @Value("${services.inventory.base-url}") String inventoryBaseUrl) {
+        this.restClient = restClient;
+        this.inventoryBaseUrl = inventoryBaseUrl;
+    }
+
+    public InventoryReservation reserve(String sku, int quantity) {
+        return restClient.get()
+                .uri(inventoryBaseUrl + "/api/inventory/{sku}?quantity={quantity}", sku, quantity)
+                .retrieve()
+                .body(InventoryReservation.class);
+    }
+}
+
--- a/order-service/src/main/java/com/oracle/demo/orders/client/PaymentClient.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/client/PaymentClient.java
@@ -0,0 +1,29 @@
+package com.oracle.demo.orders.client;
+
+import com.oracle.demo.orders.model.PaymentDecision;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.client.RestClient;
+
+import java.util.Map;
+
+@Component
+public class PaymentClient {
+
+    private final RestClient restClient;
+    private final String paymentBaseUrl;
+
+    public PaymentClient(RestClient restClient, @Value("${services.payment.base-url}") String paymentBaseUrl) {
+        this.restClient = restClient;
+        this.paymentBaseUrl = paymentBaseUrl;
+    }
+
+    public PaymentDecision authorize(String orderId, String cardToken, double amount) {
+        return restClient.post()
+                .uri(paymentBaseUrl + "/api/payments/authorize")
+                .body(Map.of("orderId", orderId, "cardToken", cardToken, "amount", amount))
+                .retrieve()
+                .body(PaymentDecision.class);
+    }
+}
+
--- a/order-service/src/main/java/com/oracle/demo/orders/model/BusinessHealth.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/model/BusinessHealth.java
@@ -0,0 +1,15 @@
+package com.oracle.demo.orders.model;
+
+public record BusinessHealth(
+        long totalOrders,
+        long confirmedOrders,
+        long pendingReviewOrders,
+        long failedOrders,
+        double manualReviewRate,
+        double totalRevenue,
+        double confirmedRevenue,
+        double revenueAtRisk,
+        boolean healthy,
+        String summary,
+        String businessImpact) {
+}
--- a/order-service/src/main/java/com/oracle/demo/orders/model/CreateOrderRequest.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/model/CreateOrderRequest.java
@@ -0,0 +1,13 @@
+package com.oracle.demo.orders.model;
+
+import jakarta.validation.constraints.Min;
+import jakarta.validation.constraints.NotBlank;
+
+public record CreateOrderRequest(
+        @NotBlank String customerId,
+        @NotBlank String sku,
+        @Min(1) int quantity,
+        @Min(1) double amount,
+        @NotBlank String cardToken) {
+}
+
--- a/order-service/src/main/java/com/oracle/demo/orders/model/InventoryReservation.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/model/InventoryReservation.java
@@ -0,0 +1,5 @@
+package com.oracle.demo.orders.model;
+
+public record InventoryReservation(String sku, boolean reserved, int availableUnits, String message) {
+}
+
--- a/order-service/src/main/java/com/oracle/demo/orders/model/OrderRecord.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/model/OrderRecord.java
@@ -0,0 +1,15 @@
+package com.oracle.demo.orders.model;
+
+import java.time.Instant;
+
+public record OrderRecord(
+        String orderId,
+        String customerId,
+        String sku,
+        int quantity,
+        double amount,
+        String status,
+        String reason,
+        Instant createdAt) {
+}
+
--- a/order-service/src/main/java/com/oracle/demo/orders/model/PaymentDecision.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/model/PaymentDecision.java
@@ -0,0 +1,5 @@
+package com.oracle.demo.orders.model;
+
+public record PaymentDecision(boolean approved, String status, String message) {
+}
+
--- a/order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilityController.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilityController.java
@@ -0,0 +1,107 @@
+package com.oracle.demo.orders.observability;
+
+import com.oracle.demo.orders.model.BusinessHealth;
+import com.oracle.demo.orders.model.OrderRecord;
+import com.oracle.demo.orders.service.OrderService;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.client.RestClient;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+@RestController
+@RequestMapping("/api/observability")
+public class ObservabilityController {
+
+    private final OrderService orderService;
+    private final RestClient restClient;
+    private final String inventoryBaseUrl;
+    private final String paymentBaseUrl;
+
+    public ObservabilityController(
+            OrderService orderService,
+            RestClient restClient,
+            @Value("${services.inventory.base-url}") String inventoryBaseUrl,
+            @Value("${services.payment.base-url}") String paymentBaseUrl) {
+        this.orderService = orderService;
+        this.restClient = restClient;
+        this.inventoryBaseUrl = inventoryBaseUrl;
+        this.paymentBaseUrl = paymentBaseUrl;
+    }
+
+    @GetMapping
+    public ObservabilitySnapshot snapshot() {
+        BusinessHealth businessHealth = orderService.businessHealth();
+        List<OrderRecord> recentOrders = orderService.listOrders().stream().limit(12).toList();
+        List<ServiceSignal> services = new ArrayList<>();
+
+        services.add(new ServiceSignal(
+                "order-service",
+                "business-api",
+                businessHealth.healthy() ? "UP" : "DEGRADED",
+                businessHealth.summary(),
+                !businessHealth.healthy()));
+        services.add(probe("inventory-service", "dependency", inventoryBaseUrl + "/actuator/health"));
+        services.add(probe("payment-service", "dependency", paymentBaseUrl + "/actuator/health"));
+
+        return new ObservabilitySnapshot(
+                Instant.now(),
+                businessHealth,
+                services,
+                recentOrders,
+                recommendedActions(businessHealth, services));
+    }
+
+    private ServiceSignal probe(String name, String layer, String url) {
+        try {
+            Map<?, ?> body = restClient.get().uri(url).retrieve().body(Map.class);
+            Object status = body == null ? null : body.get("status");
+            status = status == null ? "UNKNOWN" : status;
+            boolean healthy = "UP".equals(String.valueOf(status));
+            return new ServiceSignal(
+                    name,
+                    layer,
+                    String.valueOf(status),
+                    healthy ? "Actuator health endpoint is reachable." : "Actuator returned a non-UP status.",
+                    !healthy);
+        } catch (Exception ex) {
+            return new ServiceSignal(
+                    name,
+                    layer,
+                    "DOWN",
+                    ex.getClass().getSimpleName() + ": " + ex.getMessage(),
+                    true);
+        }
+    }
+
+    private List<String> recommendedActions(BusinessHealth businessHealth, List<ServiceSignal> services) {
+        List<String> actions = new ArrayList<>();
+
+        services.stream()
+                .filter(ServiceSignal::kagentSignal)
+                .filter(signal -> !"order-service".equals(signal.name()))
+                .findFirst()
+                .ifPresent(signal -> actions.add("Inspect " + signal.name() + " pod logs, endpoints, and recent deployment changes."));
+
+        if (!businessHealth.healthy()) {
+            if (businessHealth.revenueAtRisk() > 0) {
+                actions.add("Revenue at risk is " + businessHealth.revenueAtRisk()
+                        + " USD with manual review rate " + businessHealth.manualReviewRate()
+                        + "%. Inspect payment risk rules and recent OCI DevOps changes.");
+            }
+            actions.add("Correlate pending, failed, and revenue-at-risk signals with the latest OCI DevOps deployment.");
+            actions.add("Recommend rollback or approved config correction through OCI DevOps if degradation started after the last release.");
+        }
+
+        if (actions.isEmpty()) {
+            actions.add("No remediation required. Continue monitoring pipeline and runtime signals.");
+        }
+
+        return actions;
+    }
+}
--- a/order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilitySnapshot.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/observability/ObservabilitySnapshot.java
@@ -0,0 +1,16 @@
+package com.oracle.demo.orders.observability;
+
+import com.oracle.demo.orders.model.BusinessHealth;
+import com.oracle.demo.orders.model.OrderRecord;
+
+import java.time.Instant;
+import java.util.List;
+
+public record ObservabilitySnapshot(
+        Instant timestamp,
+        BusinessHealth businessHealth,
+        List<ServiceSignal> services,
+        List<OrderRecord> recentOrders,
+        List<String> recommendedKagentActions) {
+}
+
--- a/order-service/src/main/java/com/oracle/demo/orders/observability/ServiceSignal.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/observability/ServiceSignal.java
@@ -0,0 +1,10 @@
+package com.oracle.demo.orders.observability;
+
+public record ServiceSignal(
+        String name,
+        String layer,
+        String status,
+        String detail,
+        boolean kagentSignal) {
+}
+
--- a/order-service/src/main/java/com/oracle/demo/orders/service/OrderService.java
+++ b/order-service/src/main/java/com/oracle/demo/orders/service/OrderService.java
@@ -0,0 +1,157 @@
+package com.oracle.demo.orders.service;
+
+import com.oracle.demo.orders.client.InventoryClient;
+import com.oracle.demo.orders.client.PaymentClient;
+import com.oracle.demo.orders.model.BusinessHealth;
+import com.oracle.demo.orders.model.CreateOrderRequest;
+import com.oracle.demo.orders.model.InventoryReservation;
+import com.oracle.demo.orders.model.OrderRecord;
+import com.oracle.demo.orders.model.PaymentDecision;
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.Gauge;
+import io.micrometer.core.instrument.MeterRegistry;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+@Service
+public class OrderService {
+
+    private final InventoryClient inventoryClient;
+    private final PaymentClient paymentClient;
+    private final Map<String, OrderRecord> orders = new ConcurrentHashMap<>();
+    private final boolean stuckReviewMode;
+    private final double manualReviewRateThreshold;
+    private final double revenueAtRiskThreshold;
+    private final Counter createdCounter;
+    private final Counter reviewCounter;
+    private final Counter failedCounter;
+    private final AtomicInteger pendingReviewGauge = new AtomicInteger();
+
+    public OrderService(
+            InventoryClient inventoryClient,
+            PaymentClient paymentClient,
+            MeterRegistry meterRegistry,
+            @Value("${demo.orders.stuck-review-mode:false}") boolean stuckReviewMode,
+            @Value("${demo.orders.manual-review-rate-threshold:40}") double manualReviewRateThreshold,
+            @Value("${demo.orders.revenue-at-risk-threshold:5000}") double revenueAtRiskThreshold) {
+        this.inventoryClient = inventoryClient;
+        this.paymentClient = paymentClient;
+        this.stuckReviewMode = stuckReviewMode;
+        this.manualReviewRateThreshold = manualReviewRateThreshold;
+        this.revenueAtRiskThreshold = revenueAtRiskThreshold;
+        this.createdCounter = Counter.builder("demo_orders_created_total").register(meterRegistry);
+        this.reviewCounter = Counter.builder("demo_orders_review_required_total").register(meterRegistry);
+        this.failedCounter = Counter.builder("demo_orders_failed_total").register(meterRegistry);
+        Gauge.builder("demo_orders_pending_review", pendingReviewGauge, AtomicInteger::get).register(meterRegistry);
+    }
+
+    public OrderRecord createOrder(CreateOrderRequest request) {
+        String orderId = UUID.randomUUID().toString();
+        createdCounter.increment();
+
+        InventoryReservation inventory = inventoryClient.reserve(request.sku(), request.quantity());
+        if (!inventory.reserved()) {
+            failedCounter.increment();
+            return store(new OrderRecord(
+                    orderId,
+                    request.customerId(),
+                    request.sku(),
+                    request.quantity(),
+                    request.amount(),
+                    "FAILED",
+                    "Inventory check failed: " + inventory.message(),
+                    Instant.now()));
+        }
+
+        PaymentDecision payment = paymentClient.authorize(orderId, request.cardToken(), request.amount());
+        if (!payment.approved()) {
+            reviewCounter.increment();
+            String reason = stuckReviewMode
+                    ? "Orders are stuck in review because stuck-review-mode is enabled."
+                    : "Payment requires manual review: " + payment.message();
+            return store(new OrderRecord(
+                    orderId,
+                    request.customerId(),
+                    request.sku(),
+                    request.quantity(),
+                    request.amount(),
+                    "PENDING_REVIEW",
+                    reason,
+                    Instant.now()));
+        }
+
+        return store(new OrderRecord(
+                orderId,
+                request.customerId(),
+                request.sku(),
+                request.quantity(),
+                request.amount(),
+                "CONFIRMED",
+                "Order confirmed.",
+                Instant.now()));
+    }
+
+    public List<OrderRecord> listOrders() {
+        return orders.values().stream()
+                .sorted(Comparator.comparing(OrderRecord::createdAt).reversed())
+                .toList();
+    }
+
+    public BusinessHealth businessHealth() {
+        long total = orders.size();
+        long confirmed = orders.values().stream().filter(order -> "CONFIRMED".equals(order.status())).count();
+        long pendingReview = orders.values().stream().filter(order -> "PENDING_REVIEW".equals(order.status())).count();
+        long failed = orders.values().stream().filter(order -> "FAILED".equals(order.status())).count();
+        double totalRevenue = orders.values().stream().mapToDouble(OrderRecord::amount).sum();
+        double confirmedRevenue = orders.values().stream()
+                .filter(order -> "CONFIRMED".equals(order.status()))
+                .mapToDouble(OrderRecord::amount)
+                .sum();
+        double revenueAtRisk = orders.values().stream()
+                .filter(order -> "PENDING_REVIEW".equals(order.status()))
+                .mapToDouble(OrderRecord::amount)
+                .sum();
+        double manualReviewRate = total == 0 ? 0 : (pendingReview * 100.0) / total;
+        boolean manualReviewRateIncident = total >= 10 && manualReviewRate >= manualReviewRateThreshold;
+        boolean healthy = pendingReview < 5
+                && failed < 3
+                && !manualReviewRateIncident
+                && revenueAtRisk < revenueAtRiskThreshold;
+        String summary = healthy
+                ? "Business flow healthy."
+                : "Business degradation detected. Review pending or failed orders before promoting.";
+        String businessImpact = healthy
+                ? "No revenue protection action required."
+                : "Revenue at risk from manual-review backlog. Validate payment risk rules before continuing promotion.";
+        return new BusinessHealth(
+                total,
+                confirmed,
+                pendingReview,
+                failed,
+                round(manualReviewRate),
+                round(totalRevenue),
+                round(confirmedRevenue),
+                round(revenueAtRisk),
+                healthy,
+                summary,
+                businessImpact);
+    }
+
+    private OrderRecord store(OrderRecord order) {
+        orders.put(order.orderId(), order);
+        pendingReviewGauge.set((int) orders.values().stream().filter(item -> "PENDING_REVIEW".equals(item.status())).count());
+        return order;
+    }
+
+    private double round(double value) {
+        return Math.round(value * 100.0) / 100.0;
+    }
+}
--- a/order-service/src/main/resources/application.yml
+++ b/order-service/src/main/resources/application.yml
@@ -0,0 +1,28 @@
+spring:
+  application:
+    name: order-service
+
+server:
+  port: 8080
+
+management:
+  endpoints:
+    web:
+      exposure:
+        include: health,info,metrics,prometheus
+  endpoint:
+    health:
+      probes:
+        enabled: true
+
+services:
+  inventory:
+    base-url: http://localhost:8081
+  payment:
+    base-url: http://localhost:8082
+
+demo:
+  orders:
+    stuck-review-mode: false
+    manual-review-rate-threshold: 40
+    revenue-at-risk-threshold: 5000
--- a/order-service/src/main/resources/static/app.js
+++ b/order-service/src/main/resources/static/app.js
@@ -0,0 +1,118 @@
+const totalOrders = document.querySelector("#totalOrders");
+const confirmedOrders = document.querySelector("#confirmedOrders");
+const pendingOrders = document.querySelector("#pendingOrders");
+const failedOrders = document.querySelector("#failedOrders");
+const businessStatus = document.querySelector("#businessStatus");
+const manualReviewRate = document.querySelector("#manualReviewRate");
+const revenueAtRisk = document.querySelector("#revenueAtRisk");
+const confirmedRevenue = document.querySelector("#confirmedRevenue");
+const businessImpact = document.querySelector("#businessImpact");
+const serviceSignals = document.querySelector("#serviceSignals");
+const kagentActions = document.querySelector("#kagentActions");
+const ordersTable = document.querySelector("#ordersTable");
+const lastUpdated = document.querySelector("#lastUpdated");
+
+document.querySelector("#refreshBtn").addEventListener("click", loadDashboard);
+document.querySelector("#normalOrderBtn").addEventListener("click", () => createOrder("4111111111111111"));
+document.querySelector("#reviewOrderBtn").addEventListener("click", () => createOrder("9999123412341234"));
+
+async function createOrder(cardToken) {
+  await fetch("/api/orders", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({
+      customerId: `CUST-${Math.floor(Math.random() * 900 + 100)}`,
+      sku: "LAPTOP-15",
+      quantity: 1,
+      amount: 1499.99,
+      cardToken
+    })
+  });
+  await loadDashboard();
+}
+
+async function loadDashboard() {
+  const response = await fetch("/api/observability");
+  const snapshot = await response.json();
+  const health = snapshot.businessHealth;
+
+  totalOrders.textContent = health.totalOrders;
+  confirmedOrders.textContent = health.confirmedOrders;
+  pendingOrders.textContent = health.pendingReviewOrders;
+  failedOrders.textContent = health.failedOrders;
+  businessStatus.textContent = health.healthy ? "HEALTHY" : "DEGRADED";
+  manualReviewRate.textContent = `${formatNumber(health.manualReviewRate)}%`;
+  revenueAtRisk.textContent = formatCurrency(health.revenueAtRisk);
+  confirmedRevenue.textContent = formatCurrency(health.confirmedRevenue);
+  businessImpact.textContent = health.businessImpact;
+  lastUpdated.textContent = new Date(snapshot.timestamp).toLocaleString();
+
+  serviceSignals.innerHTML = snapshot.services.map(signal => `
+    <div class="signal">
+      <div>
+        <strong>${escapeHtml(signal.name)} <span class="muted">/ ${escapeHtml(signal.layer)}</span></strong>
+        <p>${escapeHtml(signal.detail)}</p>
+      </div>
+      <span class="status ${statusClass(signal.status)}">${escapeHtml(signal.status)}</span>
+    </div>
+  `).join("");
+
+  kagentActions.innerHTML = snapshot.recommendedKagentActions
+    .map(action => `<li>${escapeHtml(action)}</li>`)
+    .join("");
+
+  if (snapshot.recentOrders.length === 0) {
+    ordersTable.innerHTML = `<tr><td colspan="5" class="empty">No orders yet. Create a normal or review order to generate signals.</td></tr>`;
+    return;
+  }
+
+  ordersTable.innerHTML = snapshot.recentOrders.map(order => `
+    <tr>
+      <td>${escapeHtml(order.orderId.slice(0, 8))}</td>
+      <td>${escapeHtml(order.customerId)}</td>
+      <td>${escapeHtml(order.sku)}</td>
+      <td><span class="status ${statusClass(order.status)}">${escapeHtml(order.status)}</span></td>
+      <td>${escapeHtml(order.reason)}</td>
+    </tr>
+  `).join("");
+}
+
+function statusClass(status) {
+  const normalized = String(status).toLowerCase();
+  if (normalized === "up" || normalized === "confirmed" || normalized === "healthy") {
+    return "up";
+  }
+  if (normalized.includes("pending") || normalized.includes("degraded")) {
+    return "degraded";
+  }
+  if (normalized === "down" || normalized === "failed") {
+    return "down";
+  }
+  return "unknown";
+}
+
+function escapeHtml(value) {
+  return String(value ?? "")
+    .replaceAll("&", "&amp;")
+    .replaceAll("<", "&lt;")
+    .replaceAll(">", "&gt;")
+    .replaceAll('"', "&quot;")
+    .replaceAll("'", "&#039;");
+}
+
+function formatCurrency(value) {
+  return new Intl.NumberFormat("en-US", {
+    style: "currency",
+    currency: "USD",
+    maximumFractionDigits: 0
+  }).format(Number(value ?? 0));
+}
+
+function formatNumber(value) {
+  return new Intl.NumberFormat("en-US", {
+    maximumFractionDigits: 2
+  }).format(Number(value ?? 0));
+}
+
+loadDashboard();
+setInterval(loadDashboard, 10000);
--- a/order-service/src/main/resources/static/index.html
+++ b/order-service/src/main/resources/static/index.html
@@ -0,0 +1,106 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Kagent Demo Observability</title>
+  <link rel="stylesheet" href="/styles.css">
+</head>
+<body>
+  <main class="shell">
+    <section class="topbar">
+      <div>
+        <p class="section-label">OKE business application</p>
+        <h1>Order Flow Observability</h1>
+      </div>
+      <div class="actions">
+        <button id="refreshBtn" type="button">Refresh</button>
+        <button id="normalOrderBtn" type="button">Create normal order</button>
+        <button id="reviewOrderBtn" type="button" class="danger">Create review order</button>
+      </div>
+    </section>
+
+    <section class="summary-grid">
+      <article class="metric">
+        <span>Total orders</span>
+        <strong id="totalOrders">0</strong>
+      </article>
+      <article class="metric">
+        <span>Confirmed orders</span>
+        <strong id="confirmedOrders">0</strong>
+      </article>
+      <article class="metric warning">
+        <span>Pending review</span>
+        <strong id="pendingOrders">0</strong>
+      </article>
+      <article class="metric danger">
+        <span>Failed orders</span>
+        <strong id="failedOrders">0</strong>
+      </article>
+      <article class="metric">
+        <span>Business status</span>
+        <strong id="businessStatus">UNKNOWN</strong>
+      </article>
+    </section>
+
+    <section class="summary-grid business-grid">
+      <article class="metric">
+        <span>Manual review rate</span>
+        <strong id="manualReviewRate">0%</strong>
+      </article>
+      <article class="metric warning">
+        <span>Revenue at risk</span>
+        <strong id="revenueAtRisk">$0</strong>
+      </article>
+      <article class="metric">
+        <span>Confirmed revenue</span>
+        <strong id="confirmedRevenue">$0</strong>
+      </article>
+      <article class="metric">
+        <span>Business impact</span>
+        <p id="businessImpact" class="metric-note">Waiting for data</p>
+      </article>
+    </section>
+
+    <section class="workspace">
+      <article class="panel">
+        <div class="panel-head">
+          <h2>Service signals</h2>
+          <span id="lastUpdated">Waiting for data</span>
+        </div>
+        <div id="serviceSignals" class="signals"></div>
+      </article>
+
+      <article class="panel">
+        <div class="panel-head">
+          <h2>Kagent action view</h2>
+          <span>diagnostic inputs</span>
+        </div>
+        <ul id="kagentActions" class="action-list"></ul>
+      </article>
+    </section>
+
+    <section class="panel">
+      <div class="panel-head">
+        <h2>Recent orders</h2>
+        <span>business symptoms visible to the agent</span>
+      </div>
+      <div class="table-wrap">
+        <table>
+          <thead>
+            <tr>
+              <th>Order</th>
+              <th>Customer</th>
+              <th>SKU</th>
+              <th>Status</th>
+              <th>Reason</th>
+            </tr>
+          </thead>
+          <tbody id="ordersTable"></tbody>
+        </table>
+      </div>
+    </section>
+  </main>
+  <script src="/app.js"></script>
+</body>
+</html>
--- a/order-service/src/main/resources/static/styles.css
+++ b/order-service/src/main/resources/static/styles.css
@@ -0,0 +1,273 @@
+:root {
+  --bg: #f5f7fb;
+  --panel: #ffffff;
+  --panel-soft: #eef3f8;
+  --text: #15202b;
+  --muted: #5f6f82;
+  --line: #d8e0ea;
+  --accent: #126c59;
+  --accent-strong: #0f4f43;
+  --warn: #a26013;
+  --danger: #b42318;
+  --shadow: 0 18px 45px rgba(31, 45, 61, 0.10);
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  background: linear-gradient(135deg, #f8fafc 0%, #e8eef6 55%, #edf5f1 100%);
+  color: var(--text);
+  font-family: "Aptos", "Segoe UI", sans-serif;
+}
+
+button {
+  border: 0;
+  border-radius: 6px;
+  background: var(--accent);
+  color: white;
+  cursor: pointer;
+  font-size: 14px;
+  font-weight: 700;
+  padding: 11px 14px;
+}
+
+button:hover {
+  background: var(--accent-strong);
+}
+
+button.danger {
+  background: var(--danger);
+}
+
+.shell {
+  margin: 0 auto;
+  max-width: 1180px;
+  padding: 32px 20px 48px;
+}
+
+.topbar {
+  align-items: flex-end;
+  display: flex;
+  gap: 24px;
+  justify-content: space-between;
+  margin-bottom: 24px;
+}
+
+.section-label {
+  color: var(--accent);
+  font-size: 13px;
+  font-weight: 800;
+  letter-spacing: 0;
+  margin: 0 0 8px;
+  text-transform: uppercase;
+}
+
+h1,
+h2 {
+  margin: 0;
+}
+
+h1 {
+  font-size: 38px;
+  line-height: 1.05;
+}
+
+h2 {
+  font-size: 18px;
+}
+
+.actions {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 10px;
+  justify-content: flex-end;
+}
+
+.summary-grid {
+  display: grid;
+  gap: 14px;
+  grid-template-columns: repeat(4, minmax(0, 1fr));
+  margin-bottom: 16px;
+}
+
+.business-grid {
+  grid-template-columns: repeat(4, minmax(0, 1fr));
+}
+
+.metric,
+.panel {
+  background: var(--panel);
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  box-shadow: var(--shadow);
+}
+
+.metric {
+  min-height: 110px;
+  padding: 18px;
+}
+
+.metric span,
+.panel-head span {
+  color: var(--muted);
+  font-size: 13px;
+  font-weight: 700;
+}
+
+.metric strong {
+  display: block;
+  font-size: 34px;
+  margin-top: 14px;
+}
+
+.metric-note {
+  color: var(--muted);
+  font-size: 14px;
+  line-height: 1.35;
+  margin: 14px 0 0;
+}
+
+.metric.warning strong {
+  color: var(--warn);
+}
+
+.metric.danger strong {
+  color: var(--danger);
+}
+
+.workspace {
+  display: grid;
+  gap: 16px;
+  grid-template-columns: 1.25fr 0.75fr;
+  margin-bottom: 16px;
+}
+
+.panel {
+  padding: 18px;
+}
+
+.panel-head {
+  align-items: center;
+  border-bottom: 1px solid var(--line);
+  display: flex;
+  justify-content: space-between;
+  margin-bottom: 14px;
+  padding-bottom: 12px;
+}
+
+.signals {
+  display: grid;
+  gap: 10px;
+}
+
+.signal {
+  align-items: center;
+  background: var(--panel-soft);
+  border: 1px solid var(--line);
+  border-radius: 8px;
+  display: grid;
+  gap: 10px;
+  grid-template-columns: 1fr auto;
+  padding: 14px;
+}
+
+.signal strong {
+  display: block;
+  margin-bottom: 5px;
+}
+
+.muted {
+  color: var(--muted);
+  font-weight: 700;
+}
+
+.signal p {
+  color: var(--muted);
+  font-size: 13px;
+  line-height: 1.35;
+  margin: 0;
+}
+
+.status {
+  border-radius: 999px;
+  color: white;
+  font-size: 12px;
+  font-weight: 800;
+  padding: 6px 9px;
+}
+
+.status.up {
+  background: var(--accent);
+}
+
+.status.degraded,
+.status.down {
+  background: var(--danger);
+}
+
+.status.unknown {
+  background: var(--muted);
+}
+
+.action-list {
+  margin: 0;
+  padding-left: 20px;
+}
+
+.action-list li {
+  line-height: 1.45;
+  margin: 0 0 10px;
+}
+
+.table-wrap {
+  overflow-x: auto;
+}
+
+table {
+  border-collapse: collapse;
+  min-width: 860px;
+  width: 100%;
+}
+
+th,
+td {
+  border-bottom: 1px solid var(--line);
+  font-size: 14px;
+  padding: 12px 10px;
+  text-align: left;
+  vertical-align: top;
+}
+
+th {
+  color: var(--muted);
+  font-size: 12px;
+  text-transform: uppercase;
+}
+
+.empty {
+  color: var(--muted);
+  padding: 18px 10px;
+}
+
+@media (max-width: 860px) {
+  .topbar {
+    align-items: stretch;
+    flex-direction: column;
+  }
+
+  .actions {
+    justify-content: flex-start;
+  }
+
+  .summary-grid,
+  .workspace {
+    grid-template-columns: 1fr;
+  }
+
+  h1 {
+    font-size: 30px;
+  }
+}
--- a/payment-service/pom.xml
+++ b/payment-service/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.oracle.demo</groupId>
+        <artifactId>kagent-oci-devops-demo</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>payment-service</artifactId>
+    <name>payment-service</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-actuator</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-registry-prometheus</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>
--- a/payment-service/src/main/java/com/oracle/demo/payment/PaymentServiceApplication.java
+++ b/payment-service/src/main/java/com/oracle/demo/payment/PaymentServiceApplication.java
@@ -0,0 +1,13 @@
+package com.oracle.demo.payment;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class PaymentServiceApplication {
+
+    public static void main(String[] args) {
+        SpringApplication.run(PaymentServiceApplication.class, args);
+    }
+}
+
--- a/payment-service/src/main/java/com/oracle/demo/payment/api/PaymentController.java
+++ b/payment-service/src/main/java/com/oracle/demo/payment/api/PaymentController.java
@@ -0,0 +1,44 @@
+package com.oracle.demo.payment.api;
+
+import com.oracle.demo.payment.model.PaymentAuthorizationRequest;
+import com.oracle.demo.payment.model.PaymentAuthorizationResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequestMapping("/api/payments")
+public class PaymentController {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(PaymentController.class);
+
+    private final String manualReviewCardPrefix;
+    private final boolean forceTimeout;
+
+    public PaymentController(
+            @Value("${demo.payment.manual-review-card-prefix:9999}") String manualReviewCardPrefix,
+            @Value("${demo.payment.force-timeout:false}") boolean forceTimeout) {
+        this.manualReviewCardPrefix = manualReviewCardPrefix;
+        this.forceTimeout = forceTimeout;
+    }
+
+    @PostMapping("/authorize")
+    public PaymentAuthorizationResponse authorize(@RequestBody PaymentAuthorizationRequest request) throws InterruptedException {
+        if (forceTimeout) {
+            LOGGER.warn("Payment timeout mode enabled for order {}", request.orderId());
+            Thread.sleep(15000L);
+        }
+
+        if (request.cardToken() != null && request.cardToken().startsWith(manualReviewCardPrefix)) {
+            LOGGER.warn("Payment requires manual review for order {}", request.orderId());
+            return new PaymentAuthorizationResponse(false, "REVIEW_REQUIRED", "Card flagged for manual review.");
+        }
+
+        return new PaymentAuthorizationResponse(true, "APPROVED", "Payment approved.");
+    }
+}
+
--- a/payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationRequest.java
+++ b/payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationRequest.java
@@ -0,0 +1,5 @@
+package com.oracle.demo.payment.model;
+
+public record PaymentAuthorizationRequest(String orderId, String cardToken, double amount) {
+}
+
--- a/payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationResponse.java
+++ b/payment-service/src/main/java/com/oracle/demo/payment/model/PaymentAuthorizationResponse.java
@@ -0,0 +1,5 @@
+package com.oracle.demo.payment.model;
+
+public record PaymentAuthorizationResponse(boolean approved, String status, String message) {
+}
+
--- a/payment-service/src/main/resources/application.yml
+++ b/payment-service/src/main/resources/application.yml
@@ -0,0 +1,21 @@
+spring:
+  application:
+    name: payment-service
+
+server:
+  port: 8082
+
+management:
+  endpoints:
+    web:
+      exposure:
+        include: health,info,metrics,prometheus
+  endpoint:
+    health:
+      probes:
+        enabled: true
+
+demo:
+  payment:
+    manual-review-card-prefix: "9999"
+    force-timeout: false
--- a/pom.xml
+++ b/pom.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.oracle.demo</groupId>
+    <artifactId>kagent-oci-devops-demo</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+    <packaging>pom</packaging>
+
+    <name>kagent-oci-devops-demo</name>
+    <description>Spring Boot microservices demo aligned with OCI DevOps, OKE, OCI Generative AI, and kagent</description>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>4.1.0</version>
+        <relativePath/>
+    </parent>
+
+    <modules>
+        <module>inventory-service</module>
+        <module>payment-service</module>
+        <module>order-service</module>
+    </modules>
+
+    <properties>
+        <java.version>25</java.version>
+        <maven.compiler.release>${java.version}</maven.compiler.release>
+    </properties>
+</project>