adding files 2
This commit is contained in:
Oracle Public Cloud User
71
k8s/remediation/README.md
Normal file
71
k8s/remediation/README.md
Normal file
@@ -0,0 +1,71 @@
|
||||
# Declarative Remediation Manifests
|
||||
|
||||
Use these manifests with OCI DevOps `Apply manifest to your Kubernetes cluster` stages.
|
||||
|
||||
## Business remediation
|
||||
|
||||
Apply in this order:
|
||||
|
||||
```text
|
||||
business-configmap.yaml
|
||||
order-rollout-restart.yaml
|
||||
```
|
||||
|
||||
`business-configmap.yaml` sets safe business thresholds.
|
||||
|
||||
`order-rollout-restart.yaml` reapplies the `order-service` deployment template so pods reload ConfigMap-backed environment variables.
|
||||
|
||||
Before applying `order-rollout-restart.yaml`, replace:
|
||||
|
||||
```text
|
||||
<restart-token>
|
||||
```
|
||||
|
||||
with a new value, for example the OCI DevOps deployment ID, build number, or timestamp. If the annotation does not change, Kubernetes will not restart the pods.
|
||||
|
||||
## ImagePull remediation
|
||||
|
||||
Use the manifest matching the affected deployment:
|
||||
|
||||
```text
|
||||
imagepull-all-services.yaml
|
||||
imagepull-order.yaml
|
||||
imagepull-payment.yaml
|
||||
imagepull-inventory.yaml
|
||||
```
|
||||
|
||||
For the demo, prefer `imagepull-all-services.yaml`. It restores `ocir-secret` across all three business deployments, so the same remediation pipeline works regardless of which service hit `ImagePullBackOff`.
|
||||
|
||||
Each manifest restores:
|
||||
|
||||
```yaml
|
||||
imagePullSecrets:
|
||||
- name: ocir-secret
|
||||
```
|
||||
|
||||
Before applying an ImagePull remediation manifest, replace:
|
||||
|
||||
```text
|
||||
<region-key>
|
||||
<tenancy-namespace>
|
||||
<image-tag>
|
||||
<restart-token>
|
||||
```
|
||||
|
||||
with the same values used by the active deployment. The `restart-token` forces a new ReplicaSet after the secret is restored.
|
||||
|
||||
## HPA remediation
|
||||
|
||||
Use:
|
||||
|
||||
```text
|
||||
order-hpa-capacity.yaml
|
||||
```
|
||||
|
||||
It changes `order-service` HPA to:
|
||||
|
||||
```text
|
||||
minReplicas=3
|
||||
maxReplicas=8
|
||||
averageUtilization=65
|
||||
```
|
||||
14
k8s/remediation/business-configmap.yaml
Normal file
14
k8s/remediation/business-configmap.yaml
Normal file
@@ -0,0 +1,14 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: demo-config
|
||||
namespace: kagent-demo
|
||||
data:
|
||||
INVENTORY_FORCE_OUTAGE: "false"
|
||||
PAYMENT_FORCE_TIMEOUT: "false"
|
||||
PAYMENT_MANUAL_REVIEW_CARD_PREFIX: "9999"
|
||||
ORDER_STUCK_REVIEW_MODE: "false"
|
||||
ORDER_MANUAL_REVIEW_RATE_THRESHOLD: "70"
|
||||
ORDER_REVENUE_AT_RISK_THRESHOLD: "15000"
|
||||
INVENTORY_BASE_URL: "http://inventory-service:8081"
|
||||
PAYMENT_BASE_URL: "http://payment-service:8082"
|
||||
189
k8s/remediation/imagepull-all-services.yaml
Normal file
189
k8s/remediation/imagepull-all-services.yaml
Normal file
@@ -0,0 +1,189 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: order-service
|
||||
namespace: kagent-demo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: order-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: order-service
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: ocir-secret
|
||||
containers:
|
||||
- name: order-service
|
||||
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/order-service:<image-tag>
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1024Mi
|
||||
env:
|
||||
- name: SERVICES_INVENTORY_BASE_URL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: INVENTORY_BASE_URL
|
||||
- name: SERVICES_PAYMENT_BASE_URL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: PAYMENT_BASE_URL
|
||||
- name: DEMO_ORDERS_STUCK_REVIEW_MODE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: ORDER_STUCK_REVIEW_MODE
|
||||
- name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
|
||||
- name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: ORDER_REVENUE_AT_RISK_THRESHOLD
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8080
|
||||
failureThreshold: 30
|
||||
periodSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: 8080
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: payment-service
|
||||
namespace: kagent-demo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: payment-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: payment-service
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: ocir-secret
|
||||
containers:
|
||||
- name: payment-service
|
||||
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/payment-service:<image-tag>
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 8082
|
||||
env:
|
||||
- name: DEMO_PAYMENT_FORCE_TIMEOUT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: PAYMENT_FORCE_TIMEOUT
|
||||
- name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8082
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 18
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: 8082
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8082
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: inventory-service
|
||||
namespace: kagent-demo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: inventory-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: inventory-service
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: ocir-secret
|
||||
containers:
|
||||
- name: inventory-service
|
||||
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/inventory-service:<image-tag>
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 8081
|
||||
env:
|
||||
- name: DEMO_INVENTORY_FORCE_OUTAGE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: INVENTORY_FORCE_OUTAGE
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8081
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 18
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: 8081
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8081
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
53
k8s/remediation/imagepull-inventory.yaml
Normal file
53
k8s/remediation/imagepull-inventory.yaml
Normal file
@@ -0,0 +1,53 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: inventory-service
|
||||
namespace: kagent-demo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: inventory-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: inventory-service
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: ocir-secret
|
||||
containers:
|
||||
- name: inventory-service
|
||||
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/inventory-service:<image-tag>
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 8081
|
||||
env:
|
||||
- name: DEMO_INVENTORY_FORCE_OUTAGE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: INVENTORY_FORCE_OUTAGE
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8081
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 18
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: 8081
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8081
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
76
k8s/remediation/imagepull-order.yaml
Normal file
76
k8s/remediation/imagepull-order.yaml
Normal file
@@ -0,0 +1,76 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: order-service
|
||||
namespace: kagent-demo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: order-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: order-service
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: ocir-secret
|
||||
containers:
|
||||
- name: order-service
|
||||
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/order-service:<image-tag>
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1024Mi
|
||||
env:
|
||||
- name: SERVICES_INVENTORY_BASE_URL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: INVENTORY_BASE_URL
|
||||
- name: SERVICES_PAYMENT_BASE_URL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: PAYMENT_BASE_URL
|
||||
- name: DEMO_ORDERS_STUCK_REVIEW_MODE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: ORDER_STUCK_REVIEW_MODE
|
||||
- name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
|
||||
- name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: ORDER_REVENUE_AT_RISK_THRESHOLD
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8080
|
||||
failureThreshold: 30
|
||||
periodSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: 8080
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
58
k8s/remediation/imagepull-payment.yaml
Normal file
58
k8s/remediation/imagepull-payment.yaml
Normal file
@@ -0,0 +1,58 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: payment-service
|
||||
namespace: kagent-demo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: payment-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: payment-service
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: ocir-secret
|
||||
containers:
|
||||
- name: payment-service
|
||||
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/payment-service:<image-tag>
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- containerPort: 8082
|
||||
env:
|
||||
- name: DEMO_PAYMENT_FORCE_TIMEOUT
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: PAYMENT_FORCE_TIMEOUT
|
||||
- name: DEMO_PAYMENT_MANUAL_REVIEW_CARD_PREFIX
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: PAYMENT_MANUAL_REVIEW_CARD_PREFIX
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8082
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 18
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: 8082
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8082
|
||||
initialDelaySeconds: 60
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 5
|
||||
failureThreshold: 6
|
||||
22
k8s/remediation/order-hpa-capacity.yaml
Normal file
22
k8s/remediation/order-hpa-capacity.yaml
Normal file
@@ -0,0 +1,22 @@
|
||||
apiVersion: autoscaling/v2
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: order-service
|
||||
namespace: kagent-demo
|
||||
labels:
|
||||
app: order-service
|
||||
demo.oracle.com/remediation: hpa-capacity
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: order-service
|
||||
minReplicas: 3
|
||||
maxReplicas: 8
|
||||
metrics:
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
target:
|
||||
type: Utilization
|
||||
averageUtilization: 65
|
||||
77
k8s/remediation/order-rollout-restart.yaml
Normal file
77
k8s/remediation/order-rollout-restart.yaml
Normal file
@@ -0,0 +1,77 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: order-service
|
||||
namespace: kagent-demo
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: order-service
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: order-service
|
||||
annotations:
|
||||
demo.oracle.com/restart-token: "<restart-token>"
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: ocir-secret
|
||||
containers:
|
||||
- name: order-service
|
||||
image: <region-key>.ocir.io/<tenancy-namespace>/kagent-demo/order-service:latest
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 1024Mi
|
||||
env:
|
||||
- name: SERVICES_INVENTORY_BASE_URL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: INVENTORY_BASE_URL
|
||||
- name: SERVICES_PAYMENT_BASE_URL
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: PAYMENT_BASE_URL
|
||||
- name: DEMO_ORDERS_STUCK_REVIEW_MODE
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: ORDER_STUCK_REVIEW_MODE
|
||||
- name: DEMO_ORDERS_MANUAL_REVIEW_RATE_THRESHOLD
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: ORDER_MANUAL_REVIEW_RATE_THRESHOLD
|
||||
- name: DEMO_ORDERS_REVENUE_AT_RISK_THRESHOLD
|
||||
valueFrom:
|
||||
configMapKeyRef:
|
||||
name: demo-config
|
||||
key: ORDER_REVENUE_AT_RISK_THRESHOLD
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8080
|
||||
failureThreshold: 30
|
||||
periodSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/readiness
|
||||
port: 8080
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
failureThreshold: 6
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /actuator/health/liveness
|
||||
port: 8080
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 3
|
||||
Reference in New Issue
Block a user