ivanvasquez/kagent-oci-devops-demo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

adding files 2

Browse Source
This commit is contained in:
Oracle Public Cloud User
2026-09-04 13:54:16 +00:00
parent b7ec9db2bc
commit 8b3cc8dd10
83 changed files with 5045 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
k8s/base/all.yaml Normal file
View File

@@ -0,0 +1,4 @@
apiVersion: v1
kind: List
items: []

14
k8s/base/configmap.yaml Normal file
View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: demo-config
namespace: kagent-demo
data:
INVENTORY_FORCE_OUTAGE: "false"
PAYMENT_FORCE_TIMEOUT: "false"
PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999"
ORDER_STUCK_REVIEW_MODE: "false"
ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "40"
ORDER_REVENUE_AT_RISK_THRESHOLD: "5000"
INVENTORY_BASE_URL: "http://inventory-service:8081"
PAYMENT_BASE_URL: "http://payment-service:8082"

22
k8s/base/hpa.yaml Normal file
View File

@@ -0,0 +1,22 @@
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: order-service
namespace: kagent-demo
labels:
app: order-service
demo.oracle.com/remediation: hpa-capacity
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: order-service
minReplicas: 1
maxReplicas: 1
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 30

48
k8s/base/inventory.yaml Normal file
View File

@@ -0,0 +1,48 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: inventory-service
namespace: kagent-demo
spec:
replicas: 1
selector:
matchLabels:
app: inventory-service
template:
metadata:
labels:
app: inventory-service
spec:
imagePullSecrets:
- name: ocir-secret
containers:
- name: inventory-service
image: xxx.ocir.io/xxxxxxx/kagent-demo/inventory-service:2.0.0
ports:
- containerPort: 8081
env:
- name: DEMO_INVENTORY_FORCE_OUTAGE
valueFrom:
configMapKeyRef:
name: demo-config
key: INVENTORY_FORCE_OUTAGE
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8081
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8081
---
apiVersion: v1
kind: Service
metadata:
name: inventory-service
namespace: kagent-demo
spec:
selector:
app: inventory-service
ports:
- port: 8081
targetPort: 8081

5
k8s/base/namespace.yaml Normal file
View File

@@ -0,0 +1,5 @@
apiVersion: v1
kind: Namespace
metadata:
name: kagent-demo

18
k8s/base/order-loadbalancer.yaml Normal file
View File

@@ -0,0 +1,18 @@
apiVersion: v1
kind: Service
metadata:
name: order-service-lb
namespace: kagent-demo
annotations:
service.beta.kubernetes.io/oci-load-balancer-shape: flexible
service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10"
service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "10"
oci.oraclecloud.com/reserved-ips: "40.233.17.25"
spec:
type: LoadBalancer
selector:
app: order-service
ports:
- name: http
port: 80
targetPort: 8080

75
k8s/base/order.yaml Normal file
View File

@@ -0,0 +1,75 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: order-service
namespace: kagent-demo
spec:
replicas: ${REPICLAS_ORDER}
selector:
matchLabels:
app: order-service
template:
metadata:
labels:
app: order-service
spec:
imagePullSecrets:
- name: ocir-secret
containers:
- name: order-service
image: ${REGISTRY}/${NAMESPACE}/kagent-demo/order-service:${IMAGE_TAG}
ports:
- containerPort: 8080
resources:
requests:
cpu: "100m"
memory: "256Mi"
limits:
cpu: "500m"
memory: "512Mi"
env:
- name: SERVICES_INVENTORY_BASE_URL
valueFrom:
configMapKeyRef:
name: demo-config
key: INVENTORY_BASE_URL
- name: SERVICES_PAYMENT_BASE_URL
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_BASE_URL
- name: DEMO_ORDERS_STUCK_REVIEW_MODE
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_STUCK_REVIEW_MODE
- name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
- name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_REVENUE_AT_RISK_THRESHOLD
startupProbe:
httpGet:
path: /actuator/health/liveness
port: 8080
failureThreshold: 30
periodSeconds: 5
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8080
initialDelaySeconds: 10
periodSeconds: 10
failureThreshold: 6
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8080
initialDelaySeconds: 30
periodSeconds: 10
failureThreshold: 3

41
k8s/base/payment.yaml Normal file
View File

@@ -0,0 +1,41 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: payment-service
namespace: kagent-demo
spec:
replicas: 1
selector:
matchLabels:
app: payment-service
template:
metadata:
labels:
app: payment-service
spec:
imagePullSecrets:
- name: ocir-secret
containers:
- name: payment-service
image: mty.ocir.io/xxxxxxx/kagent-demo/payment-service:2.0.0
ports:
- containerPort: 8082
env:
- name: DEMO_PAYMENT_FORCE_TIMEOUT
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_FORCE_TIMEOUT
- name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8082
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8082

14
k8s/broken/business-failure-configmap.yaml Normal file
View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: demo-config
namespace: kagent-demo
data:
INVENTORY_FORCE_OUTAGE: "false"
PAYMENT_FORCE_TIMEOUT: "false"
PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999"
ORDER_STUCK_REVIEW_MODE: "true"
ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "40"
ORDER_REVENUE_AT_RISK_THRESHOLD: "5000"
INVENTORY_BASE_URL: "http://inventory-service:8081"
PAYMENT_BASE_URL: "http://payment-service:8082"

48
k8s/broken/imagepull-secret-failure-command-spec.yaml Normal file
View File

@@ -0,0 +1,48 @@
version: 0.1
component: command
timeoutInSeconds: 600
shell: bash
failImmediatelyOnError: true
env:
variables:
APP_NAMESPACE: kagent-demo
DEPLOYMENT_NAME: payment-service
CONTAINER_NAME: payment-service
BROKEN_IMAGE_PULL_SECRET: missing-ocir-secret
steps:
- type: Command
name: Simulate missing imagePullSecret
command: |
echo "Forcing ImagePullBackOff by replacing imagePullSecrets with ${BROKEN_IMAGE_PULL_SECRET}"
kubectl patch deployment "${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" \
--type merge \
-p "{
\"spec\": {
\"template\": {
\"spec\": {
\"imagePullSecrets\": [
{\"name\": \"${BROKEN_IMAGE_PULL_SECRET}\"}
],
\"containers\": [
{
\"name\": \"${CONTAINER_NAME}\",
\"imagePullPolicy\": \"Always\"
}
]
}
}
}
}"
kubectl rollout restart deployment/"${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}"
kubectl get pods -n "${APP_NAMESPACE}" -o wide
- type: Command
name: Show image pull events
command: |
sleep 30
kubectl get pods -n "${APP_NAMESPACE}" -o wide
kubectl get events -n "${APP_NAMESPACE}" --sort-by=.lastTimestamp | tail -40

22
k8s/broken/order-hpa-limited.yaml Normal file
View File

@@ -0,0 +1,22 @@
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: order-service
namespace: kagent-demo
labels:
app: order-service
demo-scenario: hpa-limited-capacity
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: order-service
minReplicas: 2
maxReplicas: 3
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 60

41
k8s/broken/payment-error.yaml Normal file
View File

@@ -0,0 +1,41 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: payment-service
namespace: kagent-demo
spec:
replicas: 1
selector:
matchLabels:
app: payment-service
template:
metadata:
labels:
app: payment-service
spec:
imagePullSecrets:
- name: ocir-secret-2
containers:
- name: payment-service
image: mty.ocir.io/xxxxxxxx/kagent-demo/payment-service:latest
ports:
- containerPort: 8082
env:
- name: DEMO_PAYMENT_FORCE_TIMEOUT
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_FORCE_TIMEOUT
- name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8082
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8082

14
k8s/broken/revenue-risk-configmap.yaml Normal file
View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: demo-config
namespace: kagent-demo
data:
INVENTORY_FORCE_OUTAGE: "false"
PAYMENT_FORCE_TIMEOUT: "false"
PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999"
ORDER_STUCK_REVIEW_MODE: "false"
ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "30"
ORDER_REVENUE_AT_RISK_THRESHOLD: "5000"
INVENTORY_BASE_URL: "http://inventory-service:8081"
PAYMENT_BASE_URL: "http://payment-service:8082"

58
k8s/broken/simulate-three-errors-command-spec.yaml Normal file
View File

@@ -0,0 +1,58 @@
version: 0.1
component: command
timeoutInSeconds: 900
shell: bash
failImmediatelyOnError: true
env:
variables:
APP_NAMESPACE: kagent-demo
PAYMENT_DEPLOYMENT: payment-service
PAYMENT_CONTAINER: payment-service
BROKEN_IMAGE_PULL_SECRET: missing-ocir-secret
steps:
- type: Command
name: Apply business degradation ConfigMap
command: |
kubectl apply -f k8s/broken/revenue-risk-configmap.yaml
kubectl rollout restart deployment/order-service -n "${APP_NAMESPACE}"
- type: Command
name: Apply limited HPA
command: |
kubectl apply -f k8s/broken/order-hpa-limited.yaml
kubectl get hpa order-service -n "${APP_NAMESPACE}" -o wide
- type: Command
name: Break payment image pull secret
command: |
kubectl patch deployment "${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" \
--type merge \
-p "{
\"spec\": {
\"template\": {
\"spec\": {
\"imagePullSecrets\": [
{\"name\": \"${BROKEN_IMAGE_PULL_SECRET}\"}
],
\"containers\": [
{
\"name\": \"${PAYMENT_CONTAINER}\",
\"imagePullPolicy\": \"Always\"
}
]
}
}
}
}"
kubectl rollout restart deployment/"${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}"
- type: Command
name: Show simulated failures
command: |
sleep 45
kubectl get pods -n "${APP_NAMESPACE}" -o wide
kubectl get hpa -n "${APP_NAMESPACE}" -o wide
kubectl get events -n "${APP_NAMESPACE}" --sort-by=.lastTimestamp | tail -60

145
k8s/delete-application-command-spec.yaml Normal file
View File

@@ -0,0 +1,145 @@
version: 0.1
component: command
timeoutInSeconds: 900
shell: bash
failImmediatelyOnError: true
env:
variables:
APP_NAMESPACE: kagent-demo
WAIT_TIMEOUT: 300s
OCI_REGION: mx-monterrey-1
OKE_CLUSTER_OCID: ocid1.cluster.oc1.mx-monterrey-1.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
OKE_ENDPOINT_TYPE: PUBLIC_ENDPOINT
steps:
- type: Command
name: Validate parameters
command: |
if [ -z "${APP_NAMESPACE}" ]; then
echo "APP_NAMESPACE parameter is required"
exit 1
fi
if [ -z "${WAIT_TIMEOUT}" ]; then
echo "WAIT_TIMEOUT parameter is required"
exit 1
fi
if [ -z "${OCI_REGION}" ]; then
echo "OCI_REGION parameter is required"
exit 1
fi
if [ -z "${OKE_CLUSTER_OCID}" ] || [ "${OKE_CLUSTER_OCID}" = "<oke-cluster-ocid>" ]; then
echo "OKE_CLUSTER_OCID parameter is required"
exit 1
fi
if [ -z "${OKE_ENDPOINT_TYPE}" ]; then
echo "OKE_ENDPOINT_TYPE parameter is required"
exit 1
fi
echo "APP_NAMESPACE=${APP_NAMESPACE}"
echo "WAIT_TIMEOUT=${WAIT_TIMEOUT}"
echo "OCI_REGION=${OCI_REGION}"
echo "OKE_CLUSTER_OCID=${OKE_CLUSTER_OCID}"
echo "OKE_ENDPOINT_TYPE=${OKE_ENDPOINT_TYPE}"
- type: Command
name: Configure kubeconfig
command: |
mkdir -p "${HOME}/.kube"
oci ce cluster create-kubeconfig \
--cluster-id "${OKE_CLUSTER_OCID}" \
--file "${HOME}/.kube/config" \
--region "${OCI_REGION}" \
--token-version 2.0.0 \
--kube-endpoint "${OKE_ENDPOINT_TYPE}"
export KUBECONFIG="${HOME}/.kube/config"
echo "KUBECONFIG=${KUBECONFIG}"
kubectl config current-context
kubectl cluster-info
- type: Command
name: Show cleanup target
command: |
echo "Deleting kagent demo application resources"
kubectl version --client=true
kubectl get namespace "${APP_NAMESPACE}"
- type: Command
name: Delete external entry point first
command: |
echo "Deleting LoadBalancer service first so OCI can release the public load balancer"
kubectl delete service order-service-lb \
-n "${APP_NAMESPACE}" \
--ignore-not-found=true
- type: Command
name: Delete application workloads
command: |
kubectl delete deployment order-service inventory-service payment-service \
-n "${APP_NAMESPACE}" \
--ignore-not-found=true
kubectl delete cronjob kagent-oke-remediator \
-n "${APP_NAMESPACE}" \
--ignore-not-found=true
kubectl delete job -l cronjob-name=kagent-oke-remediator \
-n "${APP_NAMESPACE}" \
--ignore-not-found=true
kubectl delete serviceaccount kagent-oke-remediator \
-n "${APP_NAMESPACE}" \
--ignore-not-found=true
kubectl wait --for=delete pod \
-l app=order-service \
-n "${APP_NAMESPACE}" \
--timeout="${WAIT_TIMEOUT}" || true
kubectl wait --for=delete pod \
-l app=inventory-service \
-n "${APP_NAMESPACE}" \
--timeout="${WAIT_TIMEOUT}" || true
kubectl wait --for=delete pod \
-l app=payment-service \
-n "${APP_NAMESPACE}" \
--timeout="${WAIT_TIMEOUT}" || true
kubectl wait --for=delete job \
-l cronjob-name=kagent-oke-remediator \
-n "${APP_NAMESPACE}" \
--timeout="${WAIT_TIMEOUT}" || true
- type: Command
name: Delete internal services and app config
command: |
kubectl delete service order-service inventory-service payment-service \
-n "${APP_NAMESPACE}" \
--ignore-not-found=true
kubectl delete configmap demo-config \
-n "${APP_NAMESPACE}" \
--ignore-not-found=true
kubectl delete hpa order-service inventory-service payment-service \
-n "${APP_NAMESPACE}" \
--ignore-not-found=true
- type: Command
name: Verify cleanup
command: |
echo "Remaining application resources in ${APP_NAMESPACE}:"
kubectl get deployment,svc,configmap,hpa,pod,cronjob,job,serviceaccount \
-n "${APP_NAMESPACE}" \
--ignore-not-found=true || true
echo "Application cleanup completed. Namespace and imagePullSecrets were preserved."

50
k8s/delete-namespace-command-spec.yaml Normal file
View File

@@ -0,0 +1,50 @@
version: 0.1
component: command
timeoutInSeconds: 900
shell: bash
failImmediatelyOnError: true
env:
variables:
APP_NAMESPACE: "kagent-demo"
WAIT_TIMEOUT: "300s"
steps:
- type: Command
name: Confirm target namespace
command: |
echo "Full cleanup will delete the namespace and all resources inside it."
echo "APP_NAMESPACE=${APP_NAMESPACE}"
if [ "${APP_NAMESPACE}" = "default" ] || [ "${APP_NAMESPACE}" = "kube-system" ] || [ "${APP_NAMESPACE}" = "kagent" ]; then
echo "Refusing to delete protected namespace: ${APP_NAMESPACE}"
exit 1
fi
kubectl get namespace "${APP_NAMESPACE}"
- type: Command
name: Delete LoadBalancer before namespace cleanup
command: |
echo "Deleting LoadBalancer service first to start OCI load balancer cleanup"
kubectl delete service order-service-lb \
-n "${APP_NAMESPACE}" \
--ignore-not-found=true
- type: Command
name: Delete namespace
command: |
kubectl delete namespace "${APP_NAMESPACE}" \
--ignore-not-found=true \
--timeout="${WAIT_TIMEOUT}"
- type: Command
name: Verify namespace deletion
command: |
if kubectl get namespace "${APP_NAMESPACE}" >/dev/null 2>&1; then
echo "Namespace ${APP_NAMESPACE} still exists or is terminating."
kubectl get namespace "${APP_NAMESPACE}" -o wide
exit 1
fi
echo "Namespace ${APP_NAMESPACE} was deleted successfully."

87
k8s/fix-business-config-command-spec.yaml Normal file
View File

@@ -0,0 +1,87 @@
version: 0.1
component: command
timeoutInSeconds: 900
shell: bash
failImmediatelyOnError: true
steps:
- type: Command
name: Validate remediation parameters
command: |
required_vars=(
APP_NAMESPACE
CONFIGMAP_NAME
OCI_REGION
OKE_CLUSTER_OCID
OKE_ENDPOINT_TYPE
ORDER_MANUAL_REVIEW_RATE_THRESHOLD
ORDER_REVENUE_AT_RISK_THRESHOLD
ORDER_STUCK_REVIEW_MODE
PAYMENT_FORCE_TIMEOUT
INVENTORY_FORCE_OUTAGE
WAIT_TIMEOUT
)
for var_name in "${required_vars[@]}"; do
if [ -z "${!var_name}" ]; then
echo "Required parameter is missing: ${var_name}"
exit 1
fi
done
echo "APP_NAMESPACE=${APP_NAMESPACE}"
echo "CONFIGMAP_NAME=${CONFIGMAP_NAME}"
echo "OCI_REGION=${OCI_REGION}"
echo "OKE_CLUSTER_OCID=${OKE_CLUSTER_OCID}"
echo "OKE_ENDPOINT_TYPE=${OKE_ENDPOINT_TYPE}"
echo "ORDER_MANUAL_REVIEW_RATE_THRESHOLD=${ORDER_MANUAL_REVIEW_RATE_THRESHOLD}"
echo "ORDER_REVENUE_AT_RISK_THRESHOLD=${ORDER_REVENUE_AT_RISK_THRESHOLD}"
echo "ORDER_STUCK_REVIEW_MODE=${ORDER_STUCK_REVIEW_MODE}"
echo "PAYMENT_FORCE_TIMEOUT=${PAYMENT_FORCE_TIMEOUT}"
echo "INVENTORY_FORCE_OUTAGE=${INVENTORY_FORCE_OUTAGE}"
echo "WAIT_TIMEOUT=${WAIT_TIMEOUT}"
- type: Command
name: Configure kubeconfig
command: |
mkdir -p "${HOME}/.kube"
oci ce cluster create-kubeconfig \
--cluster-id "${OKE_CLUSTER_OCID}" \
--file "${HOME}/.kube/config" \
--region "${OCI_REGION}" \
--token-version 2.0.0 \
--kube-endpoint "${OKE_ENDPOINT_TYPE}"
export KUBECONFIG="${HOME}/.kube/config"
echo "KUBECONFIG=${KUBECONFIG}"
kubectl config current-context
kubectl cluster-info
- type: Command
name: Apply business ConfigMap remediation
command: |
kubectl patch configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" \
--type merge \
-p "{
\"data\": {
\"ORDER_MANUAL_REVIEW_RATE_THRESHOLD\": \"${ORDER_MANUAL_REVIEW_RATE_THRESHOLD}\",
\"ORDER_REVENUE_AT_RISK_THRESHOLD\": \"${ORDER_REVENUE_AT_RISK_THRESHOLD}\",
\"ORDER_STUCK_REVIEW_MODE\": \"${ORDER_STUCK_REVIEW_MODE}\",
\"PAYMENT_FORCE_TIMEOUT\": \"${PAYMENT_FORCE_TIMEOUT}\",
\"INVENTORY_FORCE_OUTAGE\": \"${INVENTORY_FORCE_OUTAGE}\"
}
}"
- type: Command
name: Restart order service
command: |
kubectl rollout restart deployment/order-service -n "${APP_NAMESPACE}"
kubectl rollout status deployment/order-service -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}"
- type: Command
name: Show remediated configuration
command: |
kubectl get configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" -o yaml
kubectl get pods -n "${APP_NAMESPACE}" -o wide

66
k8s/fix-imagepull-secret-command-spec.yaml Normal file
View File

@@ -0,0 +1,66 @@
version: 0.1
component: command
timeoutInSeconds: 900
shell: bash
failImmediatelyOnError: true
env:
variables:
APP_NAMESPACE: kagent-demo
DEPLOYMENT_NAME: payment-service
CONTAINER_NAME: payment-service
IMAGE_PULL_SECRET: ocir-secret
WAIT_TIMEOUT: 300s
steps:
- type: Command
name: Restore valid imagePullSecret
command: |
case "${DEPLOYMENT_NAME}" in
order-service|payment-service|inventory-service)
echo "Target deployment is valid: ${DEPLOYMENT_NAME}"
;;
*)
echo "Invalid DEPLOYMENT_NAME=${DEPLOYMENT_NAME}"
echo "Allowed values: order-service, payment-service, inventory-service"
exit 1
;;
esac
if [ -z "${CONTAINER_NAME}" ]; then
CONTAINER_NAME="${DEPLOYMENT_NAME}"
fi
echo "APP_NAMESPACE=${APP_NAMESPACE}"
echo "DEPLOYMENT_NAME=${DEPLOYMENT_NAME}"
echo "CONTAINER_NAME=${CONTAINER_NAME}"
echo "IMAGE_PULL_SECRET=${IMAGE_PULL_SECRET}"
kubectl patch deployment "${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" \
--type merge \
-p "{
\"spec\": {
\"template\": {
\"spec\": {
\"imagePullSecrets\": [
{\"name\": \"${IMAGE_PULL_SECRET}\"}
],
\"containers\": [
{
\"name\": \"${CONTAINER_NAME}\",
\"imagePullPolicy\": \"IfNotPresent\"
}
]
}
}
}
}"
kubectl rollout restart deployment/"${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}"
kubectl rollout status deployment/"${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}"
- type: Command
name: Verify restored image pull
command: |
kubectl get deployment "${DEPLOYMENT_NAME}" -n "${APP_NAMESPACE}" -o jsonpath='{.spec.template.spec.imagePullSecrets}{"\n"}'
kubectl get pods -n "${APP_NAMESPACE}" -o wide

44
k8s/fix-order-hpa-command-spec.yaml Normal file
View File

@@ -0,0 +1,44 @@
version: 0.1
component: command
timeoutInSeconds: 600
shell: bash
failImmediatelyOnError: true
env:
variables:
APP_NAMESPACE: kagent-demo
HPA_NAME: order-service
MIN_REPLICAS: "3"
MAX_REPLICAS: "8"
CPU_TARGET: "65"
steps:
- type: Command
name: Patch order HPA capacity
command: |
kubectl patch hpa "${HPA_NAME}" -n "${APP_NAMESPACE}" \
--type merge \
-p "{
\"spec\": {
\"minReplicas\": ${MIN_REPLICAS},
\"maxReplicas\": ${MAX_REPLICAS},
\"metrics\": [
{
\"type\": \"Resource\",
\"resource\": {
\"name\": \"cpu\",
\"target\": {
\"type\": \"Utilization\",
\"averageUtilization\": ${CPU_TARGET}
}
}
}
]
}
}"
- type: Command
name: Show HPA state
command: |
kubectl get hpa "${HPA_NAME}" -n "${APP_NAMESPACE}" -o wide
kubectl describe hpa "${HPA_NAME}" -n "${APP_NAMESPACE}"

25
k8s/kagent-multi-error-payload.json Normal file
View File

@@ -0,0 +1,25 @@
{
"responseMode": "json",
"region": "${REGION}",
"appNamespace": "${APP_NAMESPACE}",
"observabilityUrl": "${ORDER_OBSERVABILITY_URL}",
"notificationTopicOcid": "${NOTIFICATION_TOPIC_OCID}",
"remediationPipelines": {
"business": {
"approvalUrl": "${BUSINESS_REMEDIATION_APPROVAL_URL}",
"projectOcid": "${BUSINESS_REMEDIATION_PROJECT_OCID}",
"pipelineOcid": "${BUSINESS_REMEDIATION_PIPELINE_OCID}"
},
"imagePull": {
"approvalUrl": "${IMAGEPULL_REMEDIATION_APPROVAL_URL}",
"projectOcid": "${IMAGEPULL_REMEDIATION_PROJECT_OCID}",
"pipelineOcid": "${IMAGEPULL_REMEDIATION_PIPELINE_OCID}"
},
"hpa": {
"approvalUrl": "${HPA_REMEDIATION_APPROVAL_URL}",
"projectOcid": "${HPA_REMEDIATION_PROJECT_OCID}",
"pipelineOcid": "${HPA_REMEDIATION_PIPELINE_OCID}"
}
},
"prompt": "Analiza el namespace kagent-demo despues del despliegue OCI DevOps. La URL de observabilidad obligatoria esta en el campo observabilityUrl del payload. Debes consultarla antes de responder y no debes preguntar por esa URL. Busca fallas activas, no eventos historicos, en estas categorias: BUSINESS_PROCESS_DEGRADATION, IMAGE_PULL_FAILURE, CRASH_LOOP_FAILURE, PROBE_FAILURE, HPA_AUTOSCALING_FAILURE, POD_SCHEDULING_FAILURE, DEPENDENCY_FAILURE, OKE_PLATFORM_FAILURE y NO_ACTIVE_FAILURE. Revisa pods, deployments, services, endpoints, eventos recientes, ConfigMap demo-config, imagePullSecrets, service accounts, HPA, logs recientes de order-service, inventory-service y payment-service, y el dashboard/API de observabilidad. Para degradacion de negocio, usa la respuesta del dashboard/API como fuente de verdad: si businessHealth.healthy=false, services incluye order-service con status DEGRADED, manualReviewRate supera el umbral configurado, pendingReviewOrders es alto o revenueAtRisk supera ORDER_REVENUE_AT_RISK_THRESHOLD, debes clasificar BUSINESS_PROCESS_DEGRADATION aunque todos los pods esten Running. Detecta si existen uno o varios problemas al mismo tiempo. Para cada problema, entrega evidencia concreta, causa raiz probable, impacto y accion recomendada para OCI DevOps. Si hay degradacion de negocio por cardToken con prefijo 9999, explica que PAYMENT_MANUAL_REVIEW_CARD_PREFIX envia esas ordenes a PENDING_REVIEW y que ORDER_MANUAL_REVIEW_RATE_THRESHOLD y ORDER_REVENUE_AT_RISK_THRESHOLD definen cuando order-service pasa a DEGRADED. Si hay degradacion de negocio, recomienda valores corregidos del ConfigMap. Si hay ImagePullBackOff o ErrImagePull, identifica deployment, imagen afectada y secreto faltante o invalido. Si hay HPA limitado, identifica si desired replicas alcanzo maxReplicas y recomienda nuevo minReplicas, maxReplicas y target CPU. Incluye un campo report para correo con title, executiveSummary, rootCause, businessImpact, evidence, configMapAnalysis y recommendedAction. El report debe ser entendible por un aprobador de OCI DevOps y debe explicar claramente por que la aplicacion esta degradada y que pipeline de remediacion debe aprobar. Responde exclusivamente JSON puro, sin markdown, sin texto antes ni despues, con este formato: {\"status\":\"HEALTHY|DEGRADED|FAILED\",\"summary\":\"\",\"report\":{\"title\":\"\",\"executiveSummary\":\"\",\"rootCause\":\"\",\"businessImpact\":\"\",\"evidence\":[],\"configMapAnalysis\":\"\",\"recommendedAction\":\"\"},\"problems\":[{\"failureType\":\"BUSINESS_PROCESS_DEGRADATION|IMAGE_PULL_FAILURE|HPA_AUTOSCALING_FAILURE|NO_ACTIVE_FAILURE\",\"rootCause\":\"\",\"evidence\":[],\"impact\":\"\",\"recommendedAction\":\"CONTINUE|APPROVAL_REQUIRED|APPLY_CONFIGMAP_FIX|FIX_IMAGE_PULL_SECRET|PATCH_HPA|ROLLBACK|REDEPLOY\",\"affectedResources\":[],\"recommendedConfigMapData\":{},\"recommendedHpaPatch\":{},\"deploymentsToRestart\":[]}],\"overallRecommendedAction\":\"CONTINUE|APPROVAL_REQUIRED|APPLY_CONFIGMAP_FIX|FIX_IMAGE_PULL_SECRET|PATCH_HPA|ROLLBACK|REDEPLOY\"}"
}

6
k8s/kagent/environment_variables.sh Normal file
View File

@@ -0,0 +1,6 @@
export OCI_GENAI_API_KEY="sk-eavZt2RSpxxxxxxxxxxxxxxxxxxxxxxxxx"
export OCI_GENAI_REGION="us-chicago-1"
export OCI_GENAI_BASE_URL="https://inference.generativeai.${OCI_GENAI_REGION}.oci.oraclecloud.com/openai/v1"
export OCI_GENAI_MODEL="openai.gpt-4o"
export OCI_GENAI_PROJECT_OCID="ocid1.generativeaiproject.oc1.us-chicago-1.xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"

28
k8s/kagent/kagent-mcp-lb.yaml Normal file
View File

@@ -0,0 +1,28 @@
apiVersion: v1
kind: Service
metadata:
name: kagent-controller-lb
namespace: kagent
annotations:
oci.oraclecloud.com/load-balancer-type: "nlb"
oci.oraclecloud.com/reserved-ips: "40.233.18.237"
labels:
app.kubernetes.io/instance: kagent
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: kagent
app.kubernetes.io/part-of: kagent
app.kubernetes.io/version: 0.9.12
helm.sh/chart: kagent-0.9.12
spec:
type: LoadBalancer
externalTrafficPolicy: Cluster
ports:
- name: mcp
port: 8083
protocol: TCP
targetPort: 8083
nodePort: 31083
selector:
app.kubernetes.io/component: controller
app.kubernetes.io/instance: kagent
app.kubernetes.io/name: kagent

27
k8s/kagent/kagent-ui-lb-exported.yaml Normal file
View File

@@ -0,0 +1,27 @@
apiVersion: v1
kind: Service
metadata:
name: kagent-ui-lb
namespace: kagent
annotations:
oci.oraclecloud.com/load-balancer-type: "nlb"
oci.oraclecloud.com/reserved-ips: "40.233.25.148"
labels:
app.kubernetes.io/instance: kagent
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: kagent
app.kubernetes.io/part-of: kagent
app.kubernetes.io/version: 0.9.12
helm.sh/chart: kagent-0.9.12
spec:
type: LoadBalancer
externalTrafficPolicy: Cluster
ports:
- port: 80
protocol: TCP
targetPort: 8080
nodePort: 31002
selector:
app.kubernetes.io/component: ui
app.kubernetes.io/instance: kagent
app.kubernetes.io/name: kagent

24
k8s/oke-remediator-multi-check/README.md Normal file
View File

@@ -0,0 +1,24 @@
# OKE Remediator Multi-Check
This workload runs every 3 minutes and performs three independent kagent checks as separate MCP interactions:
1. Business application health for Orders
2. HPA autoscaling analysis for `order-service`
3. ImagePull analysis for application deployments
Each result is normalized to a structured JSON shape. The worker stores one state entry per `error_code` in a Kubernetes ConfigMap and only sends one email plus one remediation deployment trigger per active incident signature.
## Deploy
```bash
kubectl apply -f k8s/oke-remediator-multi-check/serviceaccount.yaml
kubectl apply -f k8s/oke-remediator-multi-check/role.yaml
kubectl apply -f k8s/oke-remediator-multi-check/rolebinding.yaml
kubectl apply -f k8s/oke-remediator-multi-check/cronjob.yaml
```
## Notes
- The remediation pipelines must include approval stages.
- The notification email includes the remediation deployment approval link when the deployment is created successfully.
- The worker is intended for OKE Workload Identity.

50
k8s/oke-remediator-multi-check/cronjob.yaml Normal file
View File

@@ -0,0 +1,50 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: kagent-oke-remediator-multi
namespace: kagent-demo
spec:
schedule: "*/3 * * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 1
template:
spec:
restartPolicy: Never
serviceAccountName: kagent-oke-remediator-multi
imagePullSecrets:
- name: ocir-secret
containers:
- name: kagent-oke-remediator-multi
image: ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-multi-check:${IMAGE_TAG}
imagePullPolicy: Always
env:
- name: OCI_REGION
value: mx-monterrey-1
- name: APP_NAMESPACE
value: kagent-demo
- name: KAGENT_MCP_URL
value: http://kagent-controller.kagent.svc.cluster.local:8083/mcp
- name: KAGENT_AGENT_NAME
value: k8s-agent
- name: KAGENT_AGENT_NAMESPACE
value: kagent
- name: ORDER_OBSERVABILITY_URL
value: http://order-service-lb.kagent-demo.svc.cluster.local/api/observability
- name: NOTIFICATION_TOPIC_OCID
value: ${NOTIFICATION_TOPIC_OCID}
- name: BUSINESS_REMEDIATION_PROJECT_OCID
value: ${BUSINESS_REMEDIATION_PROJECT_OCID}
- name: BUSINESS_REMEDIATION_PIPELINE_OCID
value: ${BUSINESS_REMEDIATION_PIPELINE_OCID}
- name: IMAGEPULL_REMEDIATION_PROJECT_OCID
value: ${IMAGEPULL_REMEDIATION_PROJECT_OCID}
- name: IMAGEPULL_REMEDIATION_PIPELINE_OCID
value: ${IMAGEPULL_REMEDIATION_PIPELINE_OCID}
- name: HPA_REMEDIATION_PROJECT_OCID
value: ${HPA_REMEDIATION_PROJECT_OCID}
- name: HPA_REMEDIATION_PIPELINE_OCID
value: ${HPA_REMEDIATION_PIPELINE_OCID}

30
k8s/oke-remediator-multi-check/role.yaml Normal file
View File

@@ -0,0 +1,30 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: kagent-oke-remediator-multi
namespace: kagent-demo
rules:
- apiGroups: [""]
resources:
- pods
- events
verbs:
- get
- list
- watch
- apiGroups: ["apps"]
resources:
- deployments
- replicasets
verbs:
- get
- list
- watch
- apiGroups: [""]
resources:
- configmaps
verbs:
- get
- create
- update
- patch

13
k8s/oke-remediator-multi-check/rolebinding.yaml Normal file
View File

@@ -0,0 +1,13 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kagent-oke-remediator-multi
namespace: kagent-demo
subjects:
- kind: ServiceAccount
name: kagent-oke-remediator-multi
namespace: kagent-demo
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kagent-oke-remediator-multi

5
k8s/oke-remediator-multi-check/serviceaccount.yaml Normal file
View File

@@ -0,0 +1,5 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: kagent-oke-remediator-multi
namespace: kagent-demo

50
k8s/oke-remediator/README.md Normal file
View File

@@ -0,0 +1,50 @@
# OKE Remediator Job
This workload runs inside OKE as a `CronJob` and performs the same high-level flow as the OCI Function:
1. Queries the business observability API.
2. Sends a task to `kagent` through MCP.
3. Detects business degradation, image pull failures, or HPA capacity issues.
4. Triggers the corresponding OCI DevOps remediation deployment for approval.
5. Sends an OCI Notifications email with the diagnostic report and the approval link to the remediation deployment.
## Authentication
The workload is configured to use OKE Workload Identity first, and falls back to generic OCI resource principals only if workload identity is not available.
For OKE Workload Identity, the important pieces are:
- Enhanced OKE cluster.
- Kubernetes `ServiceAccount`.
- IAM policy scoped to:
- `request.principal.type = 'workload'`
- `request.principal.namespace = 'kagent-demo'`
- `request.principal.service_account = 'kagent-oke-remediator'`
- `request.principal.cluster_id = '<cluster-ocid>'`
No OCI config file mount is required for the manifest included here.
## Deployment
Apply:
```bash
kubectl apply -f k8s/oke-remediator/serviceaccount.yaml
kubectl apply -f k8s/oke-remediator/role.yaml
kubectl apply -f k8s/oke-remediator/rolebinding.yaml
kubectl apply -f k8s/oke-remediator/cronjob.yaml
```
Run one job immediately for validation:
```bash
kubectl create job --from=cronjob/kagent-oke-remediator kagent-oke-remediator-manual -n kagent-demo
kubectl logs job/kagent-oke-remediator-manual -n kagent-demo -f
```
## Notes
- The remediation pipelines must include an approval stage.
- The job triggers the remediation deployment and sends the email with the deployment execution link.
- The job stores the last active incident signature in the `kagent-oke-remediator-state` ConfigMap to suppress duplicate emails for the same active issue.
- The Python worker uses `oci.auth.signers.get_oke_workload_identity_resource_principal_signer()` before any fallback signer.

56
k8s/oke-remediator/cronjob.yaml Normal file
View File

@@ -0,0 +1,56 @@
apiVersion: batch/v1
kind: CronJob
metadata:
name: kagent-oke-remediator
namespace: kagent-demo
spec:
schedule: "*/3 * * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 1
template:
spec:
restartPolicy: Never
serviceAccountName: kagent-oke-remediator
imagePullSecrets:
- name: ocir-secret
containers:
- name: kagent-oke-remediator
image: ${REGISTRY}/${NAMESPACE}/kagent-demo/oke-remediator-job:${IMAGE_TAG}
imagePullPolicy: Always
env:
- name: OCI_REGION
value: mx-monterrey-1
- name: APP_NAMESPACE
value: kagent-demo
- name: KAGENT_MCP_URL
value: http://kagent-controller.kagent.svc.cluster.local:8083/mcp
- name: KAGENT_AGENT_NAME
value: k8s-agent
- name: KAGENT_AGENT_NAMESPACE
value: kagent
- name: ORDER_OBSERVABILITY_URL
value: http://order-service:8080/api/observability
- name: NOTIFICATION_TOPIC_OCID
value: ${NOTIFICATION_TOPIC_OCID}
- name: BUSINESS_REMEDIATION_PIPELINE_URL
value: ${BUSINESS_REMEDIATION_PIPELINE_URL}
- name: BUSINESS_REMEDIATION_PROJECT_OCID
value: ${BUSINESS_REMEDIATION_PROJECT_OCID}
- name: BUSINESS_REMEDIATION_PIPELINE_OCID
value: ${BUSINESS_REMEDIATION_PIPELINE_OCID}
- name: IMAGEPULL_REMEDIATION_PIPELINE_URL
value: ${IMAGEPULL_REMEDIATION_PIPELINE_URL}
- name: IMAGEPULL_REMEDIATION_PROJECT_OCID
value: ${IMAGEPULL_REMEDIATION_PROJECT_OCID}
- name: IMAGEPULL_REMEDIATION_PIPELINE_OCID
value: ${IMAGEPULL_REMEDIATION_PIPELINE_OCID}
- name: HPA_REMEDIATION_PIPELINE_URL
value: ${HPA_REMEDIATION_PIPELINE_URL}
- name: HPA_REMEDIATION_PROJECT_OCID
value: ${HPA_REMEDIATION_PROJECT_OCID}
- name: HPA_REMEDIATION_PIPELINE_OCID
value: ${HPA_REMEDIATION_PIPELINE_OCID}

9
k8s/oke-remediator/role.yaml Normal file
View File

@@ -0,0 +1,9 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: kagent-oke-remediator
namespace: kagent-demo
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "create", "update", "patch"]

13
k8s/oke-remediator/rolebinding.yaml Normal file
View File

@@ -0,0 +1,13 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: kagent-oke-remediator
namespace: kagent-demo
subjects:
- kind: ServiceAccount
name: kagent-oke-remediator
namespace: kagent-demo
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kagent-oke-remediator

5
k8s/oke-remediator/serviceaccount.yaml Normal file
View File

@@ -0,0 +1,5 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: kagent-oke-remediator
namespace: kagent-demo

94
k8s/remediate-three-errors-command-spec.yaml Normal file
View File

@@ -0,0 +1,94 @@
version: 0.1
component: command
timeoutInSeconds: 1200
shell: bash
failImmediatelyOnError: true
env:
variables:
APP_NAMESPACE: kagent-demo
CONFIGMAP_NAME: demo-config
PAYMENT_DEPLOYMENT: payment-service
PAYMENT_CONTAINER: payment-service
IMAGE_PULL_SECRET: ocir-secret
ORDER_HPA_NAME: order-service
ORDER_MIN_REPLICAS: "3"
ORDER_MAX_REPLICAS: "8"
ORDER_CPU_TARGET: "65"
WAIT_TIMEOUT: 300s
steps:
- type: Command
name: Fix business ConfigMap
command: |
kubectl patch configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" \
--type merge \
-p '{
"data": {
"ORDER_STUCK_REVIEW_MODE": "false",
"ORDER_MANUAL_REVIEW_RATE_THRESHOLD": "70",
"ORDER_REVENUE_AT_RISK_THRESHOLD": "15000",
"PAYMENT_FORCE_TIMEOUT": "false",
"INVENTORY_FORCE_OUTAGE": "false"
}
}'
kubectl rollout restart deployment/order-service -n "${APP_NAMESPACE}"
- type: Command
name: Fix payment image pull secret
command: |
kubectl patch deployment "${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" \
--type merge \
-p "{
\"spec\": {
\"template\": {
\"spec\": {
\"imagePullSecrets\": [
{\"name\": \"${IMAGE_PULL_SECRET}\"}
],
\"containers\": [
{
\"name\": \"${PAYMENT_CONTAINER}\",
\"imagePullPolicy\": \"IfNotPresent\"
}
]
}
}
}
}"
kubectl rollout restart deployment/"${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}"
- type: Command
name: Fix order HPA ceiling
command: |
kubectl patch hpa "${ORDER_HPA_NAME}" -n "${APP_NAMESPACE}" \
--type merge \
-p "{
\"spec\": {
\"minReplicas\": ${ORDER_MIN_REPLICAS},
\"maxReplicas\": ${ORDER_MAX_REPLICAS},
\"metrics\": [
{
\"type\": \"Resource\",
\"resource\": {
\"name\": \"cpu\",
\"target\": {
\"type\": \"Utilization\",
\"averageUtilization\": ${ORDER_CPU_TARGET}
}
}
}
]
}
}"
- type: Command
name: Wait and verify remediation
command: |
kubectl rollout status deployment/order-service -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}"
kubectl rollout status deployment/"${PAYMENT_DEPLOYMENT}" -n "${APP_NAMESPACE}" --timeout="${WAIT_TIMEOUT}"
kubectl get pods -n "${APP_NAMESPACE}" -o wide
kubectl get hpa -n "${APP_NAMESPACE}" -o wide
kubectl get configmap "${CONFIGMAP_NAME}" -n "${APP_NAMESPACE}" -o yaml

71
k8s/remediation/README.md Normal file
View File

@@ -0,0 +1,71 @@
# Declarative Remediation Manifests
Use these manifests with OCI DevOps `Apply manifest to your Kubernetes cluster` stages.
## Business remediation
Apply in this order:
```text
business-configmap.yaml
order-rollout-restart.yaml
```
`business-configmap.yaml` sets safe business thresholds.
`order-rollout-restart.yaml` reapplies the `order-service` deployment template so pods reload ConfigMap-backed environment variables.
Before applying `order-rollout-restart.yaml`, replace:
```text
<restart-token>
```
with a new value, for example the OCI DevOps deployment ID, build number, or timestamp. If the annotation does not change, Kubernetes will not restart the pods.
## ImagePull remediation
Use the manifest matching the affected deployment:
```text
imagepull-all-services.yaml
imagepull-order.yaml
imagepull-payment.yaml
imagepull-inventory.yaml
```
For the demo, prefer `imagepull-all-services.yaml`. It restores `ocir-secret` across all three business deployments, so the same remediation pipeline works regardless of which service hit `ImagePullBackOff`.
Each manifest restores:
```yaml
imagePullSecrets:
- name: ocir-secret
```
Before applying an ImagePull remediation manifest, replace:
```text
<region-key>
<tenancy-namespace>
<image-tag>
<restart-token>
```
with the same values used by the active deployment. The `restart-token` forces a new ReplicaSet after the secret is restored.
## HPA remediation
Use:
```text
order-hpa-capacity.yaml
```
It changes `order-service` HPA to:
```text
minReplicas=3
maxReplicas=8
averageUtilization=65
```

14
k8s/remediation/business-configmap.yaml Normal file
View File

@@ -0,0 +1,14 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: demo-config
namespace: kagent-demo
data:
INVENTORY_FORCE_OUTAGE: "false"
PAYMENT_FORCE_TIMEOUT: "false"
PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999"
ORDER_STUCK_REVIEW_MODE: "false"
ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "70"
ORDER_REVENUE_AT_RISK_THRESHOLD: "15000"
INVENTORY_BASE_URL: "http://inventory-service:8081"
PAYMENT_BASE_URL: "http://payment-service:8082"

189
k8s/remediation/imagepull-all-services.yaml Normal file
View File

@@ -0,0 +1,189 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: order-service
namespace: kagent-demo
spec:
replicas: 1
selector:
matchLabels:
app: order-service
template:
metadata:
labels:
app: order-service
spec:
imagePullSecrets:
- name: ocir-secret
containers:
- name: order-service
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/order-service:<image-tag>
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: 1000m
memory: 1024Mi
env:
- name: SERVICES_INVENTORY_BASE_URL
valueFrom:
configMapKeyRef:
name: demo-config
key: INVENTORY_BASE_URL
- name: SERVICES_PAYMENT_BASE_URL
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_BASE_URL
- name: DEMO_ORDERS_STUCK_REVIEW_MODE
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_STUCK_REVIEW_MODE
- name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
- name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_REVENUE_AT_RISK_THRESHOLD
startupProbe:
httpGet:
path: /actuator/health/liveness
port: 8080
failureThreshold: 30
periodSeconds: 5
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8080
initialDelaySeconds: 10
periodSeconds: 10
failureThreshold: 6
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8080
initialDelaySeconds: 30
periodSeconds: 10
failureThreshold: 3
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: payment-service
namespace: kagent-demo
spec:
replicas: 1
selector:
matchLabels:
app: payment-service
template:
metadata:
labels:
app: payment-service
spec:
imagePullSecrets:
- name: ocir-secret
containers:
- name: payment-service
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/payment-service:<image-tag>
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8082
env:
- name: DEMO_PAYMENT_FORCE_TIMEOUT
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_FORCE_TIMEOUT
- name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX
startupProbe:
httpGet:
path: /actuator/health/liveness
port: 8082
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 18
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8082
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8082
initialDelaySeconds: 60
periodSeconds: 20
timeoutSeconds: 5
failureThreshold: 6
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: inventory-service
namespace: kagent-demo
spec:
replicas: 1
selector:
matchLabels:
app: inventory-service
template:
metadata:
labels:
app: inventory-service
spec:
imagePullSecrets:
- name: ocir-secret
containers:
- name: inventory-service
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/inventory-service:<image-tag>
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8081
env:
- name: DEMO_INVENTORY_FORCE_OUTAGE
valueFrom:
configMapKeyRef:
name: demo-config
key: INVENTORY_FORCE_OUTAGE
startupProbe:
httpGet:
path: /actuator/health/liveness
port: 8081
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 18
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8081
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8081
initialDelaySeconds: 60
periodSeconds: 20
timeoutSeconds: 5
failureThreshold: 6

53
k8s/remediation/imagepull-inventory.yaml Normal file
View File

@@ -0,0 +1,53 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: inventory-service
namespace: kagent-demo
spec:
replicas: 1
selector:
matchLabels:
app: inventory-service
template:
metadata:
labels:
app: inventory-service
spec:
imagePullSecrets:
- name: ocir-secret
containers:
- name: inventory-service
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/inventory-service:<image-tag>
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8081
env:
- name: DEMO_INVENTORY_FORCE_OUTAGE
valueFrom:
configMapKeyRef:
name: demo-config
key: INVENTORY_FORCE_OUTAGE
startupProbe:
httpGet:
path: /actuator/health/liveness
port: 8081
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 18
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8081
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8081
initialDelaySeconds: 60
periodSeconds: 20
timeoutSeconds: 5
failureThreshold: 6

76
k8s/remediation/imagepull-order.yaml Normal file
View File

@@ -0,0 +1,76 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: order-service
namespace: kagent-demo
spec:
replicas: 1
selector:
matchLabels:
app: order-service
template:
metadata:
labels:
app: order-service
spec:
imagePullSecrets:
- name: ocir-secret
containers:
- name: order-service
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/order-service:<image-tag>
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: 1000m
memory: 1024Mi
env:
- name: SERVICES_INVENTORY_BASE_URL
valueFrom:
configMapKeyRef:
name: demo-config
key: INVENTORY_BASE_URL
- name: SERVICES_PAYMENT_BASE_URL
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_BASE_URL
- name: DEMO_ORDERS_STUCK_REVIEW_MODE
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_STUCK_REVIEW_MODE
- name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
- name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_REVENUE_AT_RISK_THRESHOLD
startupProbe:
httpGet:
path: /actuator/health/liveness
port: 8080
failureThreshold: 30
periodSeconds: 5
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8080
initialDelaySeconds: 10
periodSeconds: 10
failureThreshold: 6
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8080
initialDelaySeconds: 30
periodSeconds: 10
failureThreshold: 3

58
k8s/remediation/imagepull-payment.yaml Normal file
View File

@@ -0,0 +1,58 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: payment-service
namespace: kagent-demo
spec:
replicas: 1
selector:
matchLabels:
app: payment-service
template:
metadata:
labels:
app: payment-service
spec:
imagePullSecrets:
- name: ocir-secret
containers:
- name: payment-service
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/payment-service:<image-tag>
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8082
env:
- name: DEMO_PAYMENT_FORCE_TIMEOUT
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_FORCE_TIMEOUT
- name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX
startupProbe:
httpGet:
path: /actuator/health/liveness
port: 8082
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 18
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8082
initialDelaySeconds: 20
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 6
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8082
initialDelaySeconds: 60
periodSeconds: 20
timeoutSeconds: 5
failureThreshold: 6

22
k8s/remediation/order-hpa-capacity.yaml Normal file
View File

@@ -0,0 +1,22 @@
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
name: order-service
namespace: kagent-demo
labels:
app: order-service
demo.oracle.com/remediation: hpa-capacity
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: order-service
minReplicas: 3
maxReplicas: 8
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 65

77
k8s/remediation/order-rollout-restart.yaml Normal file
View File

@@ -0,0 +1,77 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: order-service
namespace: kagent-demo
spec:
replicas: 1
selector:
matchLabels:
app: order-service
template:
metadata:
labels:
app: order-service
annotations:
demo.oracle.com/restart-token: "<restart-token>"
spec:
imagePullSecrets:
- name: ocir-secret
containers:
- name: order-service
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/order-service:latest
ports:
- containerPort: 8080
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: 1000m
memory: 1024Mi
env:
- name: SERVICES_INVENTORY_BASE_URL
valueFrom:
configMapKeyRef:
name: demo-config
key: INVENTORY_BASE_URL
- name: SERVICES_PAYMENT_BASE_URL
valueFrom:
configMapKeyRef:
name: demo-config
key: PAYMENT_BASE_URL
- name: DEMO_ORDERS_STUCK_REVIEW_MODE
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_STUCK_REVIEW_MODE
- name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
- name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
valueFrom:
configMapKeyRef:
name: demo-config
key: ORDER_REVENUE_AT_RISK_THRESHOLD
startupProbe:
httpGet:
path: /actuator/health/liveness
port: 8080
failureThreshold: 30
periodSeconds: 5
readinessProbe:
httpGet:
path: /actuator/health/readiness
port: 8080
initialDelaySeconds: 10
periodSeconds: 10
failureThreshold: 6
livenessProbe:
httpGet:
path: /actuator/health/liveness
port: 8080
initialDelaySeconds: 30
periodSeconds: 10
failureThreshold: 3