From 321721c99160c6510fd5eadad3cb691e7ab52629 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 15:21:28 +0200
Subject: [PATCH 1/4] test(observability): require Flux reconciliation alerts

---
 scripts/validate-observability-option.sh | 48 ++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index fbf6957..8e53302 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -137,6 +137,9 @@ assert_default_off() {
   assert_resource_count "${rendered_path}" HelmRelease observability coroot-operator 0
   assert_resource_count "${rendered_path}" Coroot observability coroot 0
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
+  assert_resource_count "${rendered_path}" Provider flux-system slack 0
+  assert_resource_count "${rendered_path}" Alert flux-system platform-reconciliation 0
+  assert_resource_count "${rendered_path}" Secret flux-system slack-webhook 0
   assert_resource_count "${rendered_path}" CronJob observability cluster-heartbeat 0
   assert_resource_count "${rendered_path}" Secret observability cluster-heartbeat 0
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 0
@@ -447,6 +450,47 @@ assert_watchdog_enabled() {
   fi
 }
 
+# Verifies the production Coroot profile's exact event-driven Flux alert path.
+assert_flux_notification_contract() {
+  local rendered_path="$1"
+  local provider_contract
+  local alert_contract
+  local secret_contract
+
+  provider_contract="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.apiVersion == "notification.toolkit.fluxcd.io/v1beta3") |
+        select(.kind == "Provider" and .metadata.namespace == "flux-system" and .metadata.name == "slack") |
+        select(.spec == {"type":"slack","secretRef":{"name":"slack-webhook"}})] | length'
+  )"
+  alert_contract="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.apiVersion == "notification.toolkit.fluxcd.io/v1beta3") |
+        select(.kind == "Alert" and .metadata.namespace == "flux-system" and .metadata.name == "platform-reconciliation") |
+        select(.spec == {
+          "providerRef":{"name":"slack"},
+          "eventSeverity":"error",
+          "eventSources":[{"kind":"Kustomization","name":"*"}],
+          "summary":"Flux reconciliation error — production platform"
+        })] | length'
+  )"
+  secret_contract="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.apiVersion == "v1") |
+        select(.kind == "Secret" and .metadata.namespace == "flux-system" and .metadata.name == "slack-webhook") |
+        select(.type == "Opaque") |
+        select(.stringData == {"address":"${alertmanager_webhook_url:=https://example.invalid/no-slack-configured}"})] | length'
+  )"
+
+  if [[ "${provider_contract}" != "1" || "${alert_contract}" != "1" || "${secret_contract}" != "1" ]]; then
+    echo "production Coroot must render one exact Flux Kustomization error path through the existing Slack webhook" >&2
+    return 1
+  fi
+}
+
 # Validates that only the observability namespace's generated policies defer
 # the dedicated heartbeat label to its narrower workload policy.
 assert_heartbeat_policy_exclusion() {
@@ -976,11 +1020,15 @@ done
 assert_resource_count "${docker_infrastructure}" HelmRelease observability coroot-operator 0
 assert_resource_count "${docker_infrastructure}" Coroot observability coroot 1
 assert_resource_count "${docker_infrastructure}" HelmRelease observability audit-log-forwarder 0
+assert_resource_count "${docker_infrastructure}" Provider flux-system slack 0
+assert_resource_count "${docker_infrastructure}" Alert flux-system platform-reconciliation 0
+assert_resource_count "${docker_infrastructure}" Secret flux-system slack-webhook 0
 assert_opencost_absent "${docker_infrastructure}"
 
 assert_resource_count "${hetzner_infrastructure}" HelmRelease observability coroot-operator 0
 assert_resource_count "${hetzner_infrastructure}" Coroot observability coroot 1
 assert_resource_count "${hetzner_infrastructure}" HelmRelease observability audit-log-forwarder 1
+assert_flux_notification_contract "${hetzner_infrastructure}"
 assert_opencost_absent "${hetzner_infrastructure}"
 assert_opencost_absent "${docker_apps}"
 assert_opencost_absent "${hetzner_apps}"

From 5a576042454b2f330eb481abb135772d513fbfaa Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 15:35:23 +0200
Subject: [PATCH 2/4] feat(observability): restore Flux reconciliation alerts

---
 docs/TEMPLATING.md                              |  8 ++++++--
 docs/dr/alerting.md                             | 15 ++++++++++++---
 .../flux-notifications/alert.yaml               | 17 +++++++++++++++++
 .../flux-notifications/kustomization.yaml       |  7 +++++++
 .../flux-notifications/provider.yaml            | 11 +++++++++++
 .../flux-notifications/secret.yaml              | 11 +++++++++++
 .../infrastructure-coroot/kustomization.yaml    |  4 ++++
 scripts/validate-observability-option.sh        | 10 ++++++++--
 8 files changed, 76 insertions(+), 7 deletions(-)
 create mode 100644 k8s/providers/hetzner/infrastructure-coroot/flux-notifications/alert.yaml
 create mode 100644 k8s/providers/hetzner/infrastructure-coroot/flux-notifications/kustomization.yaml
 create mode 100644 k8s/providers/hetzner/infrastructure-coroot/flux-notifications/provider.yaml
 create mode 100644 k8s/providers/hetzner/infrastructure-coroot/flux-notifications/secret.yaml

diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index 6a5afc9..f55f823 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -86,6 +86,9 @@ enforced for that namespace.
 The production / Hetzner Coroot profile reuses the existing encrypted webhook
 variable for Coroot incident and resolution notifications. Its matching provider
 policy permits only `hooks.slack.com:443`, so keep this value a Slack incoming-webhook URL.
+Flux notification-controller reuses the same `alertmanager_webhook_url` only in
+the production Coroot profile to report failed `Kustomization` reconciliations;
+there is no fourth secret or polling workload to configure.
 Per-alert notifications remain visible only in the Coroot UI; this avoids sending
 every noisy alert to the shared webhook. Local / Docker Coroot stays notification-free.
 The profile also stages one hardened `cluster-heartbeat` CronJob for the cluster
@@ -109,8 +112,9 @@ deny policy from blocking the new workload during an upgrade. The invalid URL
 fallback keeps local and unconfigured instances inert. Both default and Coroot
 profiles keep Watchdog unchanged on the distinct Alertmanager pipeline monitor,
 which preserves a dead-man signal until the apps layer starts the direct check.
-Kube-prometheus-stack keeps owning its remaining alert
-rules, including Flux reconciliation alerts, until that migration is delivered
+Flux notification-controller now owns reconciliation errors in the production
+Coroot profile. Kube-prometheus-stack keeps owning its remaining metric-backed
+alert rules and the Watchdog pipeline until those migrations are delivered
 separately.
 
 Open the Coroot UI at `https://observability.<your-domain>`. Community Edition
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index 2a52c98..f853069 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -76,8 +76,13 @@ keeps an unconfigured CronJob quiet.
 The bearer URL reaches the CronJob through a Secret-backed environment variable.
 The shell feeds it to curl as stdin configuration, so the credential is absent
 from both the PodSpec and curl's runtime argument vector.
-Flux reconciliation alerting still depends on kube-prometheus-stack, so that stack
-remains transitional until a later slice replaces those rules before removing it.
+Flux notification-controller now owns reconciliation errors in the production
+Coroot profile. One event-driven `Alert` watches every `Kustomization` at `error`
+severity, covering failed builds, applies, dependencies, and readiness waits. It
+posts through the existing `alertmanager_webhook_url`; the default and local
+Coroot profiles render no Slack `Provider`, `Alert`, or webhook `Secret`.
+`kube-prometheus-stack` remains transitional for Watchdog and the remaining
+metric-backed alert rules until those paths are migrated separately.
 
 Recommended monitor: [healthchecks.io](https://healthchecks.io) (free,
 open-source, native Slack integration). Keep two distinct checks: the existing
@@ -142,6 +147,10 @@ Two sources:
 
 (plus the chart's workload/storage/self-monitoring alerts.)
 
+In the production Coroot profile, the event-driven Flux notification above
+replaces `FluxKustomizationNotReady`; the other table entries still come from
+`kube-prometheus-stack` while the profile is transitional.
+
 ## Per-environment setup (manual SOPS steps)
 
 The three secret inputs — `alertmanager_webhook_url`,
@@ -151,7 +160,7 @@ be set by hand. All three are read from the `Secret` `variables-cluster`, which
 is a Flux `substituteFrom` source.
 
 ```bash
-# 1. Slack incoming webhook for alert notifications.
+# 1. Slack incoming webhook for Alertmanager, Coroot, and Flux notifications.
 sops --set '["stringData"]["alertmanager_webhook_url"] "https://hooks.slack.com/services/XXX/YYY/ZZZ"' \
   k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
 
diff --git a/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/alert.yaml b/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/alert.yaml
new file mode 100644
index 0000000..a4afce0
--- /dev/null
+++ b/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/alert.yaml
@@ -0,0 +1,17 @@
+# Coroot sees workload health, but it cannot observe failed Flux builds,
+# applies, dependencies, or readiness waits. Every top-level platform layer is
+# a wait-gated Kustomization, so error events cover that complete failure class
+# without introducing another polling workload.
+apiVersion: notification.toolkit.fluxcd.io/v1beta3
+kind: Alert
+metadata:
+  name: platform-reconciliation
+  namespace: flux-system
+spec:
+  providerRef:
+    name: slack
+  eventSeverity: error
+  eventSources:
+    - kind: Kustomization
+      name: "*"
+  summary: "Flux reconciliation error — production platform"
diff --git a/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/kustomization.yaml b/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/kustomization.yaml
new file mode 100644
index 0000000..de378c3
--- /dev/null
+++ b/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/kustomization.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - secret.yaml
+  - provider.yaml
+  - alert.yaml
diff --git a/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/provider.yaml b/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/provider.yaml
new file mode 100644
index 0000000..d9efefe
--- /dev/null
+++ b/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/provider.yaml
@@ -0,0 +1,11 @@
+# notification-controller is already enabled by the shared Flux installation;
+# this provider only connects it to the production Coroot webhook input.
+apiVersion: notification.toolkit.fluxcd.io/v1beta3
+kind: Provider
+metadata:
+  name: slack
+  namespace: flux-system
+spec:
+  type: slack
+  secretRef:
+    name: slack-webhook
diff --git a/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/secret.yaml b/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/secret.yaml
new file mode 100644
index 0000000..116b0de
--- /dev/null
+++ b/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/secret.yaml
@@ -0,0 +1,11 @@
+# Reuse the Coroot incident webhook so Flux reconciliation errors reach the
+# same operator channel without another secret to provision. The placeholder
+# keeps static validation usable when an instance has not decrypted its input.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: slack-webhook
+  namespace: flux-system
+type: Opaque
+stringData:
+  address: ${alertmanager_webhook_url:=https://example.invalid/no-slack-configured}
diff --git a/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml b/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml
index 08c7cbd..1158e65 100644
--- a/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml
+++ b/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml
@@ -5,6 +5,10 @@ resources:
   - ../infrastructure/
   - ../../../bases/infrastructure/coroot/
   - ../../../bases/infrastructure/audit-log-forwarder/
+  # Coroot observes workloads but cannot see a failed Flux build, apply, or
+  # readiness wait. Restore that production safety net through Flux's existing
+  # notification controller and the same webhook input Coroot already uses.
+  - flux-notifications/
 components:
   - ../../../components/observability-coroot/allow-host-access/
   - ../../../components/observability-coroot/isolate-heartbeat/
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 8e53302..6916d66 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -1185,12 +1185,15 @@ for documented_boundary in \
   "preserves a dead-man signal" \
   "keeps kube-prometheus-stack" \
   "Cost allocation is therefore unavailable" \
+  "reuses the same \`alertmanager_webhook_url\`" \
+  "production Coroot profile to report failed" \
   "reuses the existing encrypted webhook" \
   "incident and resolution notifications" \
   "permits only \`hooks.slack.com:443\`" \
   "Per-alert notifications remain visible only in the Coroot UI" \
   "Local / Docker Coroot stays notification-free" \
-  "Kube-prometheus-stack keeps owning its remaining alert" \
+  "Flux notification-controller now owns reconciliation errors" \
+  "Kube-prometheus-stack keeps owning its remaining metric-backed" \
   "https://observability.<your-domain>" \
   "Dex-backed oauth2-proxy" \
   "no direct Gateway route to the Coroot service" \
@@ -1222,7 +1225,10 @@ for documented_heartbeat_boundary in \
   "dependent apps layer" \
   "while Watchdog preserves" \
   "curl as stdin configuration" \
-  "Flux reconciliation alerting still depends on kube-prometheus-stack"; do
+  "Flux notification-controller now owns reconciliation errors" \
+  "watches every \`Kustomization\`" \
+  "Coroot profiles render no Slack" \
+  "\`kube-prometheus-stack\` remains transitional for Watchdog"; do
   if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then
     echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2
     exit 1

From 6f69aeb9deb8412f99f523ef1a6ab1a85cd6632c Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 17:24:52 +0200
Subject: [PATCH 3/4] fix: harden Flux reconciliation alerts

---
 docs/TEMPLATING.md                            | 15 ++--
 docs/dr/alerting.md                           | 21 +++--
 .../alert-dependency-wait.yaml                | 18 +++++
 .../flux-notifications/alert.yaml             |  8 +-
 .../flux-notifications/kustomization.yaml     |  1 +
 .../flux-notifications/provider.yaml          |  0
 .../flux-notifications/secret.yaml            |  0
 .../kustomization.yaml                        |  3 +
 .../infrastructure-coroot/kustomization.yaml  | 11 ++-
 .../patches/retain-helm-release-alert.yaml    | 24 ++++++
 scripts/validate-observability-option.sh      | 79 +++++++++++++++++--
 11 files changed, 152 insertions(+), 28 deletions(-)
 create mode 100644 k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert-dependency-wait.yaml
 rename k8s/providers/hetzner/{infrastructure-coroot => infrastructure-controllers-coroot}/flux-notifications/alert.yaml (54%)
 rename k8s/providers/hetzner/{infrastructure-coroot => infrastructure-controllers-coroot}/flux-notifications/kustomization.yaml (80%)
 rename k8s/providers/hetzner/{infrastructure-coroot => infrastructure-controllers-coroot}/flux-notifications/provider.yaml (100%)
 rename k8s/providers/hetzner/{infrastructure-coroot => infrastructure-controllers-coroot}/flux-notifications/secret.yaml (100%)
 create mode 100644 k8s/providers/hetzner/infrastructure-coroot/patches/retain-helm-release-alert.yaml

diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index f55f823..b137469 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -87,8 +87,13 @@ The production / Hetzner Coroot profile reuses the existing encrypted webhook
 variable for Coroot incident and resolution notifications. Its matching provider
 policy permits only `hooks.slack.com:443`, so keep this value a Slack incoming-webhook URL.
 Flux notification-controller reuses the same `alertmanager_webhook_url` only in
-the production Coroot profile to report failed `Kustomization` reconciliations;
-there is no fourth secret or polling workload to configure.
+the production Coroot profile to report failed reconciliations. The
+controller layer applies both Flux alerts before it waits for every controller
+to become ready: one forwards all `Kustomization` error events, while a
+narrow info-severity path for dependency waits covers a prerequisite that is
+not ready. The legacy Prometheus rule keeps only HelmRelease coverage, so a
+Kustomization failure reaches Slack once. There
+is no fourth secret or polling workload to configure.
 Per-alert notifications remain visible only in the Coroot UI; this avoids sending
 every noisy alert to the shared webhook. Local / Docker Coroot stays notification-free.
 The profile also stages one hardened `cluster-heartbeat` CronJob for the cluster
@@ -113,9 +118,9 @@ fallback keeps local and unconfigured instances inert. Both default and Coroot
 profiles keep Watchdog unchanged on the distinct Alertmanager pipeline monitor,
 which preserves a dead-man signal until the apps layer starts the direct check.
 Flux notification-controller now owns reconciliation errors in the production
-Coroot profile. Kube-prometheus-stack keeps owning its remaining metric-backed
-alert rules and the Watchdog pipeline until those migrations are delivered
-separately.
+Coroot profile, including dependency waits.
+Kube-prometheus-stack keeps owning its remaining metric-backed alert rules and
+the Watchdog pipeline until those migrations are delivered separately.
 
 Open the Coroot UI at `https://observability.<your-domain>`. Community Edition
 does not provide native OIDC, so this profile sends every UI request through the
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index f853069..9f10cf2 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -77,10 +77,14 @@ The bearer URL reaches the CronJob through a Secret-backed environment variable.
 The shell feeds it to curl as stdin configuration, so the credential is absent
 from both the PodSpec and curl's runtime argument vector.
 Flux notification-controller now owns reconciliation errors in the production
-Coroot profile. One event-driven `Alert` watches every `Kustomization` at `error`
-severity, covering failed builds, applies, dependencies, and readiness waits. It
-posts through the existing `alertmanager_webhook_url`; the default and local
-Coroot profiles render no Slack `Provider`, `Alert`, or webhook `Secret`.
+Coroot profile. Two event-driven `Alert` objects watch every `Kustomization`:
+one forwards `error` events for failed builds, applies, and health checks; the
+other forwards only `info` dependency waits whose prerequisite is not ready.
+The controller layer applies them before it waits for every controller, so the
+notification path already exists if that layer cannot become ready. Both post
+through the existing `alertmanager_webhook_url`.
+The default and local Coroot profiles render no Slack `Provider`, `Alert`, or
+webhook `Secret`.
 `kube-prometheus-stack` remains transitional for Watchdog and the remaining
 metric-backed alert rules until those paths are migrated separately.
 
@@ -147,9 +151,12 @@ Two sources:
 
 (plus the chart's workload/storage/self-monitoring alerts.)
 
-In the production Coroot profile, the event-driven Flux notification above
-replaces `FluxKustomizationNotReady`; the other table entries still come from
-`kube-prometheus-stack` while the profile is transitional.
+In the production Coroot profile, the event-driven Flux notifications replace
+the Kustomization half of `FluxKustomizationNotReady`. Its profile patch renames
+the remaining metric rule to `FluxHelmReleaseNotReady` and narrows it to
+HelmRelease readiness, so the two paths do not send the same failure twice. The
+other table entries still come from `kube-prometheus-stack` while the profile is
+transitional.
 
 ## Per-environment setup (manual SOPS steps)
 
diff --git a/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert-dependency-wait.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert-dependency-wait.yaml
new file mode 100644
index 0000000..2a5333e
--- /dev/null
+++ b/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert-dependency-wait.yaml
@@ -0,0 +1,18 @@
+# Dependency waits are info events in Flux. Match only their message shape so
+# other successful or routine info events do not create operator noise.
+apiVersion: notification.toolkit.fluxcd.io/v1beta3
+kind: Alert
+metadata:
+  name: platform-dependency-wait
+  namespace: flux-system
+spec:
+  providerRef:
+    name: slack
+  eventSeverity: info
+  eventSources:
+    - kind: Kustomization
+      name: "*"
+  inclusionList:
+    - ".*dependency.*not ready.*"
+  eventMetadata:
+    summary: "Flux dependency wait — production platform"
diff --git a/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/alert.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert.yaml
similarity index 54%
rename from k8s/providers/hetzner/infrastructure-coroot/flux-notifications/alert.yaml
rename to k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert.yaml
index a4afce0..921f745 100644
--- a/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/alert.yaml
+++ b/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert.yaml
@@ -1,7 +1,6 @@
 # Coroot sees workload health, but it cannot observe failed Flux builds,
-# applies, dependencies, or readiness waits. Every top-level platform layer is
-# a wait-gated Kustomization, so error events cover that complete failure class
-# without introducing another polling workload.
+# applies, or health checks. Error events cover those failure classes without
+# introducing another polling workload.
 apiVersion: notification.toolkit.fluxcd.io/v1beta3
 kind: Alert
 metadata:
@@ -14,4 +13,5 @@ spec:
   eventSources:
     - kind: Kustomization
       name: "*"
-  summary: "Flux reconciliation error — production platform"
+  eventMetadata:
+    summary: "Flux reconciliation error — production platform"
diff --git a/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/kustomization.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/kustomization.yaml
similarity index 80%
rename from k8s/providers/hetzner/infrastructure-coroot/flux-notifications/kustomization.yaml
rename to k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/kustomization.yaml
index de378c3..ea1b9dc 100644
--- a/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/kustomization.yaml
+++ b/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/kustomization.yaml
@@ -5,3 +5,4 @@ resources:
   - secret.yaml
   - provider.yaml
   - alert.yaml
+  - alert-dependency-wait.yaml
diff --git a/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/provider.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/provider.yaml
similarity index 100%
rename from k8s/providers/hetzner/infrastructure-coroot/flux-notifications/provider.yaml
rename to k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/provider.yaml
diff --git a/k8s/providers/hetzner/infrastructure-coroot/flux-notifications/secret.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/secret.yaml
similarity index 100%
rename from k8s/providers/hetzner/infrastructure-coroot/flux-notifications/secret.yaml
rename to k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/secret.yaml
diff --git a/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
index 3688282..482bd73 100644
--- a/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
+++ b/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
@@ -4,6 +4,9 @@ kind: Kustomization
 resources:
   - ../infrastructure/controllers/
   - ../../../bases/infrastructure/controllers/coroot/
+  # Apply the production notification path before this Flux layer waits for
+  # every controller, so it can report a controller readiness failure itself.
+  - flux-notifications/
 components:
   - ../../../components/observability-coroot/remove-controllers-loki-alloy/
   - ../../../components/observability-coroot/remove-controller-opencost/
diff --git a/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml b/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml
index 1158e65..912b869 100644
--- a/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml
+++ b/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml
@@ -5,10 +5,6 @@ resources:
   - ../infrastructure/
   - ../../../bases/infrastructure/coroot/
   - ../../../bases/infrastructure/audit-log-forwarder/
-  # Coroot observes workloads but cannot see a failed Flux build, apply, or
-  # readiness wait. Restore that production safety net through Flux's existing
-  # notification controller and the same webhook input Coroot already uses.
-  - flux-notifications/
 components:
   - ../../../components/observability-coroot/allow-host-access/
   - ../../../components/observability-coroot/isolate-heartbeat/
@@ -21,3 +17,10 @@ patches:
       kind: Coroot
       name: coroot
       namespace: observability
+  - path: patches/retain-helm-release-alert.yaml
+    target:
+      group: monitoring.coreos.com
+      version: v1
+      kind: PrometheusRule
+      name: platform-critical
+      namespace: monitoring
diff --git a/k8s/providers/hetzner/infrastructure-coroot/patches/retain-helm-release-alert.yaml b/k8s/providers/hetzner/infrastructure-coroot/patches/retain-helm-release-alert.yaml
new file mode 100644
index 0000000..8c3407d
--- /dev/null
+++ b/k8s/providers/hetzner/infrastructure-coroot/patches/retain-helm-release-alert.yaml
@@ -0,0 +1,24 @@
+- op: test
+  path: /spec/groups/3/name
+  value: flux
+- op: test
+  path: /spec/groups/3/rules/0/alert
+  value: FluxKustomizationNotReady
+- op: test
+  path: /spec/groups/3/rules/0/expr
+  value: |
+    gotk_reconcile_condition{type="Ready",status="False",
+                             kind=~"Kustomization|HelmRelease"} == 1
+- op: replace
+  path: /spec/groups/3/rules/0/alert
+  value: FluxHelmReleaseNotReady
+- op: replace
+  path: /spec/groups/3/rules/0/expr
+  value: |
+    gotk_reconcile_condition{type="Ready",status="False",kind="HelmRelease"} == 1
+- op: replace
+  path: /spec/groups/3/rules/0/annotations/summary
+  value: "Flux HelmRelease {{ $labels.namespace }}/{{ $labels.name }} not Ready"
+- op: replace
+  path: /spec/groups/3/rules/0/annotations/description
+  value: "kubectl describe HelmRelease -n {{ $labels.namespace }} {{ $labels.name }}"
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 6916d66..2ca36ac 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -139,6 +139,7 @@ assert_default_off() {
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
   assert_resource_count "${rendered_path}" Provider flux-system slack 0
   assert_resource_count "${rendered_path}" Alert flux-system platform-reconciliation 0
+  assert_resource_count "${rendered_path}" Alert flux-system platform-dependency-wait 0
   assert_resource_count "${rendered_path}" Secret flux-system slack-webhook 0
   assert_resource_count "${rendered_path}" CronJob observability cluster-heartbeat 0
   assert_resource_count "${rendered_path}" Secret observability cluster-heartbeat 0
@@ -454,7 +455,8 @@ assert_watchdog_enabled() {
 assert_flux_notification_contract() {
   local rendered_path="$1"
   local provider_contract
-  local alert_contract
+  local error_alert_contract
+  local dependency_alert_contract
   local secret_contract
 
   provider_contract="$(
@@ -464,7 +466,7 @@ assert_flux_notification_contract() {
         select(.kind == "Provider" and .metadata.namespace == "flux-system" and .metadata.name == "slack") |
         select(.spec == {"type":"slack","secretRef":{"name":"slack-webhook"}})] | length'
   )"
-  alert_contract="$(
+  error_alert_contract="$(
     yq eval-all -o=json '.' "${rendered_path}" |
       jq -s '[.[] |
         select(.apiVersion == "notification.toolkit.fluxcd.io/v1beta3") |
@@ -473,7 +475,20 @@ assert_flux_notification_contract() {
           "providerRef":{"name":"slack"},
           "eventSeverity":"error",
           "eventSources":[{"kind":"Kustomization","name":"*"}],
-          "summary":"Flux reconciliation error — production platform"
+          "eventMetadata":{"summary":"Flux reconciliation error — production platform"}
+        })] | length'
+  )"
+  dependency_alert_contract="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.apiVersion == "notification.toolkit.fluxcd.io/v1beta3") |
+        select(.kind == "Alert" and .metadata.namespace == "flux-system" and .metadata.name == "platform-dependency-wait") |
+        select(.spec == {
+          "providerRef":{"name":"slack"},
+          "eventSeverity":"info",
+          "eventSources":[{"kind":"Kustomization","name":"*"}],
+          "inclusionList":[".*dependency.*not ready.*"],
+          "eventMetadata":{"summary":"Flux dependency wait — production platform"}
         })] | length'
   )"
   secret_contract="$(
@@ -485,8 +500,39 @@ assert_flux_notification_contract() {
         select(.stringData == {"address":"${alertmanager_webhook_url:=https://example.invalid/no-slack-configured}"})] | length'
   )"
 
-  if [[ "${provider_contract}" != "1" || "${alert_contract}" != "1" || "${secret_contract}" != "1" ]]; then
-    echo "production Coroot must render one exact Flux Kustomization error path through the existing Slack webhook" >&2
+  if [[ "${provider_contract}" != "1" || "${error_alert_contract}" != "1" || "${dependency_alert_contract}" != "1" || "${secret_contract}" != "1" ]]; then
+    echo "production Coroot controllers must render exact Flux error and dependency-wait paths through the existing Slack webhook" >&2
+    return 1
+  fi
+}
+
+# Ensures the production Coroot profile does not send Kustomization failures
+# through both Flux notification-controller and the legacy Prometheus rule.
+assert_legacy_flux_alert_deduplicated() {
+  local rendered_path="$1"
+  local matches
+
+  matches="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.apiVersion == "monitoring.coreos.com/v1") |
+        select(.kind == "PrometheusRule" and .metadata.namespace == "monitoring" and .metadata.name == "platform-critical") |
+        .spec.groups[] |
+        select(.name == "flux") |
+        select(.rules == [{
+          "alert":"FluxHelmReleaseNotReady",
+          "expr":"gotk_reconcile_condition{type=\"Ready\",status=\"False\",kind=\"HelmRelease\"} == 1\n",
+          "for":"15m",
+          "labels":{"severity":"critical"},
+          "annotations":{
+            "summary":"Flux HelmRelease {{ $labels.namespace }}/{{ $labels.name }} not Ready",
+            "description":"kubectl describe HelmRelease -n {{ $labels.namespace }} {{ $labels.name }}"
+          }
+        }])] | length'
+  )"
+
+  if [[ "${matches}" != "1" ]]; then
+    echo "production Coroot must retain only HelmRelease coverage in the legacy Flux Prometheus rule" >&2
     return 1
   fi
 }
@@ -981,6 +1027,12 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   fi
 done
 
+assert_resource_count "${docker_controllers}" Provider flux-system slack 0
+assert_resource_count "${docker_controllers}" Alert flux-system platform-reconciliation 0
+assert_resource_count "${docker_controllers}" Alert flux-system platform-dependency-wait 0
+assert_resource_count "${docker_controllers}" Secret flux-system slack-webhook 0
+assert_flux_notification_contract "${hetzner_controllers}"
+
 assert_heartbeat_staged_after_policy_exclusion "${docker_controllers}" "${docker_apps}"
 assert_heartbeat_staged_after_policy_exclusion "${hetzner_controllers}" "${hetzner_apps}"
 
@@ -1022,13 +1074,18 @@ assert_resource_count "${docker_infrastructure}" Coroot observability coroot 1
 assert_resource_count "${docker_infrastructure}" HelmRelease observability audit-log-forwarder 0
 assert_resource_count "${docker_infrastructure}" Provider flux-system slack 0
 assert_resource_count "${docker_infrastructure}" Alert flux-system platform-reconciliation 0
+assert_resource_count "${docker_infrastructure}" Alert flux-system platform-dependency-wait 0
 assert_resource_count "${docker_infrastructure}" Secret flux-system slack-webhook 0
 assert_opencost_absent "${docker_infrastructure}"
 
 assert_resource_count "${hetzner_infrastructure}" HelmRelease observability coroot-operator 0
 assert_resource_count "${hetzner_infrastructure}" Coroot observability coroot 1
 assert_resource_count "${hetzner_infrastructure}" HelmRelease observability audit-log-forwarder 1
-assert_flux_notification_contract "${hetzner_infrastructure}"
+assert_resource_count "${hetzner_infrastructure}" Provider flux-system slack 0
+assert_resource_count "${hetzner_infrastructure}" Alert flux-system platform-reconciliation 0
+assert_resource_count "${hetzner_infrastructure}" Alert flux-system platform-dependency-wait 0
+assert_resource_count "${hetzner_infrastructure}" Secret flux-system slack-webhook 0
+assert_legacy_flux_alert_deduplicated "${hetzner_infrastructure}"
 assert_opencost_absent "${hetzner_infrastructure}"
 assert_opencost_absent "${docker_apps}"
 assert_opencost_absent "${hetzner_apps}"
@@ -1193,6 +1250,9 @@ for documented_boundary in \
   "Per-alert notifications remain visible only in the Coroot UI" \
   "Local / Docker Coroot stays notification-free" \
   "Flux notification-controller now owns reconciliation errors" \
+  "controller layer applies both Flux alerts" \
+  "narrow info-severity path for dependency waits" \
+  "legacy Prometheus rule keeps only HelmRelease coverage" \
   "Kube-prometheus-stack keeps owning its remaining metric-backed" \
   "https://observability.<your-domain>" \
   "Dex-backed oauth2-proxy" \
@@ -1226,8 +1286,11 @@ for documented_heartbeat_boundary in \
   "while Watchdog preserves" \
   "curl as stdin configuration" \
   "Flux notification-controller now owns reconciliation errors" \
-  "watches every \`Kustomization\`" \
-  "Coroot profiles render no Slack" \
+  "Two event-driven \`Alert\` objects" \
+  "The controller layer applies them before it waits" \
+  "\`FluxHelmReleaseNotReady\`" \
+  "watch every \`Kustomization\`" \
+  "default and local Coroot profiles render no Slack" \
   "\`kube-prometheus-stack\` remains transitional for Watchdog"; do
   if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then
     echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2

From 68c42e29e83294a0497756815754babb31e39077 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 19:19:01 +0200
Subject: [PATCH 4/4] fix: match Flux dependency wait events

---
 .../alert-dependency-wait.yaml                  |  2 +-
 scripts/validate-observability-option.sh        | 17 ++++++++++-------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert-dependency-wait.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert-dependency-wait.yaml
index 2a5333e..f26947d 100644
--- a/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert-dependency-wait.yaml
+++ b/k8s/providers/hetzner/infrastructure-controllers-coroot/flux-notifications/alert-dependency-wait.yaml
@@ -13,6 +13,6 @@ spec:
     - kind: Kustomization
       name: "*"
   inclusionList:
-    - ".*dependency.*not ready.*"
+    - ".*Dependencies do not meet ready condition.*"
   eventMetadata:
     summary: "Flux dependency wait — production platform"
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 2ca36ac..4e05bc4 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -483,13 +483,16 @@ assert_flux_notification_contract() {
       jq -s '[.[] |
         select(.apiVersion == "notification.toolkit.fluxcd.io/v1beta3") |
         select(.kind == "Alert" and .metadata.namespace == "flux-system" and .metadata.name == "platform-dependency-wait") |
-        select(.spec == {
-          "providerRef":{"name":"slack"},
-          "eventSeverity":"info",
-          "eventSources":[{"kind":"Kustomization","name":"*"}],
-          "inclusionList":[".*dependency.*not ready.*"],
-          "eventMetadata":{"summary":"Flux dependency wait — production platform"}
-        })] | length'
+        select(.spec as $spec |
+          $spec == {
+            "providerRef":{"name":"slack"},
+            "eventSeverity":"info",
+            "eventSources":[{"kind":"Kustomization","name":"*"}],
+            "inclusionList":[".*Dependencies do not meet ready condition.*"],
+            "eventMetadata":{"summary":"Flux dependency wait — production platform"}
+          } and
+          ("Dependencies do not meet ready condition, retrying in 5s" | test($spec.inclusionList[0])) and
+          ("Reconciliation finished in 250ms, next run in 10m0s" | test($spec.inclusionList[0]) | not))] | length'
   )"
   secret_contract="$(
     yq eval-all -o=json '.' "${rendered_path}" |