From e77d360ebfb08b7617a0154a6016ecc56071e3e9 Mon Sep 17 00:00:00 2001
From: David AI <david-ai@localhost>
Date: Mon, 6 Jul 2026 14:50:21 +0000
Subject: [PATCH] sample-app: add PDB and RollingUpdate strategy to contain bad
 rollouts

- Deployment.spec.strategy: maxUnavailable=0, maxSurge=1 so a bad rollout
  stalls on failing readiness instead of replacing healthy pods
- PodDisruptionBudget (maxUnavailable=1) so voluntary disruptions can't
  stack with an in-progress bad rollout and take the whole fleet down
- runbook.md: documents both, and flags the still-open gap (alert
  runbook_url + automated rollback) as out of scope for this repo
---
 apps/sample-app/base/deployment.yaml    |  5 +++++
 apps/sample-app/base/kustomization.yaml |  1 +
 apps/sample-app/base/pdb.yaml           |  9 +++++++++
 docs/runbook.md                         | 18 ++++++++++++++++++
 4 files changed, 33 insertions(+)
 create mode 100644 apps/sample-app/base/pdb.yaml

diff --git a/apps/sample-app/base/deployment.yaml b/apps/sample-app/base/deployment.yaml
index 5f8e262..64ca1ff 100644
--- a/apps/sample-app/base/deployment.yaml
+++ b/apps/sample-app/base/deployment.yaml
@@ -9,6 +9,11 @@ spec:
   selector:
     matchLabels:
       app: sample-app
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 1
   template:
     metadata:
       labels:
diff --git a/apps/sample-app/base/kustomization.yaml b/apps/sample-app/base/kustomization.yaml
index 60a7a0c..b7bb48b 100644
--- a/apps/sample-app/base/kustomization.yaml
+++ b/apps/sample-app/base/kustomization.yaml
@@ -5,3 +5,4 @@ resources:
   - service.yaml
   - hpa.yaml
   - networkpolicy.yaml
+  - pdb.yaml
diff --git a/apps/sample-app/base/pdb.yaml b/apps/sample-app/base/pdb.yaml
new file mode 100644
index 0000000..ec312b9
--- /dev/null
+++ b/apps/sample-app/base/pdb.yaml
@@ -0,0 +1,9 @@
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: sample-app
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      app: sample-app
diff --git a/docs/runbook.md b/docs/runbook.md
index 1597fa9..26ce29d 100644
--- a/docs/runbook.md
+++ b/docs/runbook.md
@@ -14,3 +14,21 @@ for anything deploy-related, or the ArgoCD UI for sync/health state.
 | Need to add/change a new environment | Copy an existing overlay under `apps/sample-app/overlays/`, add matching ArgoCD `Application` under `argocd/apps/`, add it to `argocd/app-of-apps.yaml` | Keep `namespace:` unique per env |
 | Bootstrapping ArgoCD on a fresh cluster | Follow "Bootstrap ArgoCD" section in `README.md` | |
 | Rotating/adding a secret the app needs | Not yet supported — no secrets management is wired up in this repo yet (tracked separately, likely External Secrets Operator + SSM Parameter Store) | Don't hand-roll a Secret manifest into git in the meantime |
+
+## Bad rollout containment
+
+- **Rollout strategy**: `Deployment.spec.strategy` is `RollingUpdate` with
+  `maxUnavailable: 0, maxSurge: 1` — a bad rollout won't take down existing
+  healthy pods until the new ones pass readiness. Combined with the readiness
+  probe, a crash-looping new version stalls the rollout instead of replacing
+  the whole fleet.
+- **PodDisruptionBudget**: `maxUnavailable: 1` on `sample-app` protects
+  against voluntary disruptions (node drains, cluster-autoscaler, manual
+  `kubectl delete`) stacking with an in-progress bad rollout and taking out
+  every pod at once. In `dev` (1 replica) this still allows full disruption
+  when the single pod is intentionally evicted — that's expected there.
+- **Not yet wired**: the `PodCrashLooping` alert (in `cluster-observability`)
+  doesn't yet carry a `runbook_url` annotation pointing back here, and there's
+  no automated rollback (no Argo Rollouts analysis step) — today, detection
+  is automatic but response is still a human reading this doc. Out of scope
+  for this repo/PR; tracked separately.