From 01d265776bdc67c19bc26deaae293ec1fd7cc455 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 21:24:34 +0200
Subject: [PATCH 01/26] test(observability): require Coroot heartbeat

---
 scripts/validate-observability-option.sh | 60 ++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 5350ba5..1759f8a 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -106,10 +106,68 @@ assert_default_off() {
   assert_resource_count "${rendered_path}" HelmRelease observability coroot-operator 0
   assert_resource_count "${rendered_path}" Coroot observability coroot 0
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
+  assert_resource_count "${rendered_path}" CronJob observability cluster-heartbeat 0
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 0
   assert_resource_count "${rendered_path}" HTTPRoute observability coroot 0
 }
 
+assert_coroot_heartbeat_contract() {
+  local rendered_path="$1"
+  local cronjob_contract
+  local heartbeat_egress_contract
+  local broad_external_egress
+
+  cronjob_contract="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.kind == "CronJob" and .metadata.namespace == "observability" and .metadata.name == "cluster-heartbeat") |
+        select(.spec.schedule == "*/5 * * * *") |
+        select(.spec.concurrencyPolicy == "Forbid") |
+        select(.spec.startingDeadlineSeconds == 30) |
+        select(.spec.successfulJobsHistoryLimit == 1 and .spec.failedJobsHistoryLimit == 1) |
+        select(.spec.jobTemplate.spec.backoffLimit == 0 and .spec.jobTemplate.spec.activeDeadlineSeconds == 120) |
+        .spec.jobTemplate.spec.template.spec as $pod |
+        select($pod.restartPolicy == "Never" and $pod.automountServiceAccountToken == false) |
+        select($pod.securityContext.runAsNonRoot == true and $pod.securityContext.runAsUser == 65532) |
+        select($pod.securityContext.seccompProfile.type == "RuntimeDefault") |
+        select($pod.containers | length == 1) |
+        $pod.containers[0] as $container |
+        select($container.name == "heartbeat") |
+        select($container.image == "docker.io/curlimages/curl:8.21.0@sha256:7c12af72ceb38b7432ab85e1a265cff6ae58e06f95539d539b654f2cfa64bb13") |
+        select($container.command[0:2] == ["/bin/sh", "-c"]) |
+        select($container.command[2] | contains("${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}")) |
+        select($container.command[2] | contains("|| true")) |
+        select($container.securityContext.allowPrivilegeEscalation == false) |
+        select($container.securityContext.readOnlyRootFilesystem == true) |
+        select($container.securityContext.runAsNonRoot == true and $container.securityContext.runAsUser == 65532 and $container.securityContext.runAsGroup == 65532) |
+        select($container.securityContext.capabilities.drop == ["ALL"])] | length'
+  )"
+  heartbeat_egress_contract="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-coroot") |
+        .spec.egress[]? |
+        select(.toFQDNs == [{"matchName":"hc-ping.com"}]) |
+        select(.toPorts == [{"ports":[{"port":"443","protocol":"TCP"}]}])] | length'
+  )"
+  broad_external_egress="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-coroot") |
+        .spec.egress[]? |
+        select(any(.toEntities[]?; . == "world" or . == "all") or any(.toFQDNs[]?; has("matchPattern")))] | length'
+  )"
+
+  if [[ "${cronjob_contract}" != "1" ]]; then
+    echo "Coroot profile must render one hardened five-minute cluster heartbeat in ${rendered_path}" >&2
+    return 1
+  fi
+  if [[ "${heartbeat_egress_contract}" != "1" || "${broad_external_egress}" != "0" ]]; then
+    echo "Coroot heartbeat must have one exact hc-ping.com:443 egress path without wildcard or world access in ${rendered_path}" >&2
+    return 1
+  fi
+}
+
 assert_opencost_present() {
   local rendered_path="$1"
 
@@ -494,9 +552,11 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_resource_count "${rendered_path}" HelmRelease observability coroot-operator 1
   assert_resource_count "${rendered_path}" Coroot observability coroot 0
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
+  assert_resource_count "${rendered_path}" HelmRelease monitoring kube-prometheus-stack 1
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1
   assert_opencost_absent "${rendered_path}"
   assert_coroot_sso_controller_contract "${rendered_path}"
+  assert_coroot_heartbeat_contract "${rendered_path}"
 done
 
 for rendered_path in "${docker_infrastructure}" "${hetzner_infrastructure}"; do

From b8eb8823b1abfbfc2bc70b2fa218a30d957d71f1 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 21:32:07 +0200
Subject: [PATCH 02/26] feat(observability): stage Coroot heartbeat

---
 docs/TEMPLATING.md                            |  8 ++-
 docs/dr/alerting.md                           | 17 +++--
 .../coroot/cilium-network-policy.yaml         |  9 +++
 .../controllers/coroot/cron-job.yaml          | 67 +++++++++++++++++++
 .../controllers/coroot/kustomization.yaml     |  1 +
 scripts/validate-observability-option.sh      | 16 +++++
 6 files changed, 111 insertions(+), 7 deletions(-)
 create mode 100644 k8s/bases/infrastructure/controllers/coroot/cron-job.yaml

diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index 500dc85..ea460e1 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -88,8 +88,12 @@ variable for Coroot incident and resolution notifications. Its matching provider
 policy permits only `hooks.slack.com:443`, so keep this value a Slack incoming-webhook URL.
 Per-alert notifications remain visible only in the Coroot UI; this avoids sending
 every noisy alert to the shared webhook. Local / Docker Coroot stays notification-free.
-Kube-prometheus-stack keeps owning its alert rules until that remaining metrics
-and alerting migration is delivered separately.
+The profile also stages one hardened `cluster-heartbeat` CronJob for the cluster
+dead-man signal. It reuses the existing encrypted heartbeat URL and does not add a new secret.
+Its policy permits only `hc-ping.com:443`; the invalid fallback keeps local and
+unconfigured instances inert. Kube-prometheus-stack keeps owning its alert rules,
+including Flux reconciliation alerts, until that remaining metrics and alerting
+migration is delivered separately.
 
 Open the Coroot UI at `https://observability.<your-domain>`. Community Edition
 does not provide native OIDC, so this profile sends every UI request through the
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index 9ba8401..c9a5939 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -51,11 +51,18 @@ clusters get an invalid URL and stay quiet by design.
 ## Dead-man's-switch (off-cluster heartbeat)
 
 In-cluster Alertmanager cannot tell you the cluster is down — it's down
-too. To cover that, the chart's always-firing `Watchdog` alert is routed
-to a dedicated `heartbeat` receiver that POSTs to an **external** monitor
-on a tight cadence (`repeat_interval: 50s`). If the cluster — or the
-Prometheus → Alertmanager pipeline — dies, the monitor stops receiving
-pings and notifies Slack out-of-band.
+too. The default profile keeps the `Watchdog` alert and routes it to a
+dedicated `heartbeat` receiver that POSTs to an **external** monitor on a
+tight cadence (`repeat_interval: 50s`). If the cluster — or the Prometheus →
+Alertmanager pipeline — dies, the monitor stops receiving pings and notifies
+Slack out-of-band.
+
+The Coroot profile uses the dedicated `cluster-heartbeat` CronJob instead. It
+pings the same existing URL every five minutes, independently of Coroot, and
+its network policy allows only `hc-ping.com:443`. The invalid fallback keeps
+local and unconfigured instances quiet. Flux reconciliation alerting still depends on kube-prometheus-stack,
+so that stack remains transitional until a later slice replaces those rules
+before removing it.
 
 Recommended monitor: [healthchecks.io](https://healthchecks.io) (free,
 open-source, native Slack integration). Create a check with a ~5 min
diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
index d775742..4788d9f 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
@@ -41,6 +41,15 @@ spec:
         - ports:
             - port: "443"
               protocol: TCP
+    # The profile's cluster dead-man switch pings only the external heartbeat
+    # service. Keep this separate from Coroot's own upstream allow-list so the
+    # heartbeat boundary stays exact and independently mutation-testable.
+    - toFQDNs:
+        - matchName: hc-ping.com
+      toPorts:
+        - ports:
+            - port: "443"
+              protocol: TCP
     # DNS resolution also enables Cilium's FQDN proxy for the allow-list above.
     - toEndpoints:
         - matchLabels:
diff --git a/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml b/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
new file mode 100644
index 0000000..db8541c
--- /dev/null
+++ b/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
@@ -0,0 +1,67 @@
+# Cluster-level dead-man's-switch. Every five minutes a pod pings the
+# per-cluster external heartbeat monitor. The monitor alerts out of band when
+# pings stop, covering the one failure that in-cluster alerting cannot report:
+# the whole cluster being unavailable.
+#
+# The ping is unconditional. Component health remains Coroot's responsibility;
+# coupling this signal to Coroot would turn a component outage into a false
+# cluster-down alarm.
+#
+# The URL comes from the existing Flux substitution input. Local and
+# unconfigured clusters use the invalid fallback, and `|| true` keeps the Job
+# from flapping when no external monitor is configured. The signal is the
+# absence of pings at the external monitor, not the Job exit code.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cluster-heartbeat
+  namespace: observability
+spec:
+  schedule: "*/5 * * * *"
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 30
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 120
+      template:
+        metadata:
+          labels:
+            app: cluster-heartbeat
+        spec:
+          restartPolicy: Never
+          automountServiceAccountToken: false
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: heartbeat
+              image: docker.io/curlimages/curl:8.21.0@sha256:7c12af72ceb38b7432ab85e1a265cff6ae58e06f95539d539b654f2cfa64bb13
+              command:
+                - /bin/sh
+                - -c
+                - >-
+                  curl -sf --max-time 10
+                  --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused
+                  "${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}"
+                  || true
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                runAsNonRoot: true
+                runAsUser: 65532
+                runAsGroup: 65532
+                capabilities:
+                  drop:
+                    - ALL
+              resources:
+                requests:
+                  cpu: 5m
+                  memory: 16Mi
+                limits:
+                  cpu: 50m
+                  memory: 32Mi
diff --git a/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml b/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
index 59132d1..17eff79 100644
--- a/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
@@ -6,6 +6,7 @@ kind: Kustomization
 resources:
   - namespace.yaml
   - cilium-network-policy.yaml
+  - cron-job.yaml
   - helm-repository.yaml
   - helm-release.yaml
   - http-route.yaml
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 1759f8a..b7a8188 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -719,6 +719,10 @@ for documented_boundary in \
   "This profile is transitional." \
   "removes OpenCost and its Headlamp" \
   "retires the legacy Loki and Alloy log path" \
+  "stages one hardened \`cluster-heartbeat\` CronJob" \
+  "reuses the existing encrypted heartbeat URL" \
+  "permits only \`hc-ping.com:443\`" \
+  "does not add a new secret" \
   "keeps kube-prometheus-stack" \
   "Cost allocation is therefore unavailable" \
   "reuses the existing encrypted webhook" \
@@ -737,6 +741,18 @@ for documented_boundary in \
   fi
 done
 
+alerting_guide="${repo_root}/docs/dr/alerting.md"
+for documented_heartbeat_boundary in \
+  "The default profile keeps the \`Watchdog\` alert" \
+  "The Coroot profile uses the dedicated \`cluster-heartbeat\` CronJob" \
+  "hc-ping.com:443" \
+  "Flux reconciliation alerting still depends on kube-prometheus-stack"; do
+  if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then
+    echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2
+    exit 1
+  fi
+done
+
 if grep -R -E -n '(devantler|homelab|hooks\.slack\.com|hcloud|alertmanager_webhook_url)' \
   "${repo_root}/k8s/bases/infrastructure/controllers/coroot" \
   "${repo_root}/k8s/bases/infrastructure/coroot" \

From a44465348d0094575e485ef0d54987997332ec96 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 21:35:08 +0200
Subject: [PATCH 03/26] test(observability): tighten heartbeat contract

---
 scripts/validate-observability-option.sh | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index b7a8188..4c63544 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -126,7 +126,9 @@ assert_coroot_heartbeat_contract() {
         select(.spec.startingDeadlineSeconds == 30) |
         select(.spec.successfulJobsHistoryLimit == 1 and .spec.failedJobsHistoryLimit == 1) |
         select(.spec.jobTemplate.spec.backoffLimit == 0 and .spec.jobTemplate.spec.activeDeadlineSeconds == 120) |
-        .spec.jobTemplate.spec.template.spec as $pod |
+        .spec.jobTemplate.spec.template as $template |
+        select($template.metadata.labels.app == "cluster-heartbeat") |
+        $template.spec as $pod |
         select($pod.restartPolicy == "Never" and $pod.automountServiceAccountToken == false) |
         select($pod.securityContext.runAsNonRoot == true and $pod.securityContext.runAsUser == 65532) |
         select($pod.securityContext.seccompProfile.type == "RuntimeDefault") |
@@ -134,13 +136,19 @@ assert_coroot_heartbeat_contract() {
         $pod.containers[0] as $container |
         select($container.name == "heartbeat") |
         select($container.image == "docker.io/curlimages/curl:8.21.0@sha256:7c12af72ceb38b7432ab85e1a265cff6ae58e06f95539d539b654f2cfa64bb13") |
-        select($container.command[0:2] == ["/bin/sh", "-c"]) |
-        select($container.command[2] | contains("${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}")) |
-        select($container.command[2] | contains("|| true")) |
+        select($container.command == [
+          "/bin/sh",
+          "-c",
+          "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}\" || true"
+        ]) |
         select($container.securityContext.allowPrivilegeEscalation == false) |
         select($container.securityContext.readOnlyRootFilesystem == true) |
         select($container.securityContext.runAsNonRoot == true and $container.securityContext.runAsUser == 65532 and $container.securityContext.runAsGroup == 65532) |
-        select($container.securityContext.capabilities.drop == ["ALL"])] | length'
+        select($container.securityContext.capabilities.drop == ["ALL"]) |
+        select($container.resources == {
+          "requests":{"cpu":"5m","memory":"16Mi"},
+          "limits":{"cpu":"50m","memory":"32Mi"}
+        })] | length'
   )"
   heartbeat_egress_contract="$(
     yq eval-all -o=json '.' "${rendered_path}" |

From 972fc87b04e98fd09c17002c0e986f85ee49b57b Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 21:46:30 +0200
Subject: [PATCH 04/26] fix(observability): isolate heartbeat egress

---
 .../cilium-network-policy-heartbeat.yaml      | 30 +++++++++++++++++++
 .../coroot/cilium-network-policy.yaml         | 16 ++++------
 .../controllers/coroot/kustomization.yaml     |  1 +
 scripts/validate-observability-option.sh      | 29 +++++++++++++-----
 4 files changed, 59 insertions(+), 17 deletions(-)
 create mode 100644 k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml

diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
new file mode 100644
index 0000000..0b20f80
--- /dev/null
+++ b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
@@ -0,0 +1,30 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-cluster-heartbeat
+  namespace: observability
+spec:
+  endpointSelector:
+    matchLabels:
+      app: cluster-heartbeat
+  egress:
+    - toFQDNs:
+        - matchName: hc-ping.com
+      toPorts:
+        - ports:
+            - port: "443"
+              protocol: TCP
+    # DNS is the only other egress path required to resolve the exact endpoint.
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+            - port: "53"
+              protocol: TCP
+          rules:
+            dns:
+              - matchPattern: "*"
diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
index 4788d9f..b066859 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
@@ -4,7 +4,12 @@ metadata:
   name: allow-coroot
   namespace: observability
 spec:
-  endpointSelector: {}
+  endpointSelector:
+    matchExpressions:
+      - key: app
+        operator: NotIn
+        values:
+          - cluster-heartbeat
   ingress:
     # Coroot, its bundled stores, its agents, and the audit forwarder exchange
     # telemetry inside this namespace.
@@ -41,15 +46,6 @@ spec:
         - ports:
             - port: "443"
               protocol: TCP
-    # The profile's cluster dead-man switch pings only the external heartbeat
-    # service. Keep this separate from Coroot's own upstream allow-list so the
-    # heartbeat boundary stays exact and independently mutation-testable.
-    - toFQDNs:
-        - matchName: hc-ping.com
-      toPorts:
-        - ports:
-            - port: "443"
-              protocol: TCP
     # DNS resolution also enables Cilium's FQDN proxy for the allow-list above.
     - toEndpoints:
         - matchLabels:
diff --git a/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml b/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
index 17eff79..a1d40a5 100644
--- a/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
@@ -5,6 +5,7 @@ kind: Kustomization
 # establish the CRD before Flux server-side dry-runs the custom resource.
 resources:
   - namespace.yaml
+  - cilium-network-policy-heartbeat.yaml
   - cilium-network-policy.yaml
   - cron-job.yaml
   - helm-repository.yaml
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 4c63544..04b6198 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -108,6 +108,7 @@ assert_default_off() {
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
   assert_resource_count "${rendered_path}" CronJob observability cluster-heartbeat 0
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 0
+  assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 0
   assert_resource_count "${rendered_path}" HTTPRoute observability coroot 0
 }
 
@@ -115,6 +116,7 @@ assert_coroot_heartbeat_contract() {
   local rendered_path="$1"
   local cronjob_contract
   local heartbeat_egress_contract
+  local coroot_selector_contract
   local broad_external_egress
 
   cronjob_contract="$(
@@ -151,12 +153,24 @@ assert_coroot_heartbeat_contract() {
         })] | length'
   )"
   heartbeat_egress_contract="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-cluster-heartbeat") |
+        select(.spec.endpointSelector.matchLabels == {"app":"cluster-heartbeat"}) |
+        select(.spec.egress | length == 2) |
+        select(any(.spec.egress[];
+          .toFQDNs == [{"matchName":"hc-ping.com"}] and
+          .toPorts == [{"ports":[{"port":"443","protocol":"TCP"}]}])) |
+        select(any(.spec.egress[];
+          .toEndpoints == [{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}] and
+          .toPorts == [{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}],"rules":{"dns":[{"matchPattern":"*"}]}}]))] | length'
+  )"
+  coroot_selector_contract="$(
     yq eval-all -o=json '.' "${rendered_path}" |
       jq -s '[.[] |
         select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-coroot") |
-        .spec.egress[]? |
-        select(.toFQDNs == [{"matchName":"hc-ping.com"}]) |
-        select(.toPorts == [{"ports":[{"port":"443","protocol":"TCP"}]}])] | length'
+        select(.spec.endpointSelector.matchExpressions == [{"key":"app","operator":"NotIn","values":["cluster-heartbeat"]}]) |
+        select(all(.spec.egress[]?.toFQDNs[]?; .matchName != "hc-ping.com"))] | length'
   )"
   broad_external_egress="$(
     yq eval-all -o=json '.' "${rendered_path}" |
@@ -170,8 +184,8 @@ assert_coroot_heartbeat_contract() {
     echo "Coroot profile must render one hardened five-minute cluster heartbeat in ${rendered_path}" >&2
     return 1
   fi
-  if [[ "${heartbeat_egress_contract}" != "1" || "${broad_external_egress}" != "0" ]]; then
-    echo "Coroot heartbeat must have one exact hc-ping.com:443 egress path without wildcard or world access in ${rendered_path}" >&2
+  if [[ "${heartbeat_egress_contract}" != "1" || "${coroot_selector_contract}" != "1" || "${broad_external_egress}" != "0" ]]; then
+    echo "Coroot heartbeat must be excluded from the shared policy and have one exact hc-ping.com:443 egress path without wildcard or world access in ${rendered_path}" >&2
     return 1
   fi
 }
@@ -560,8 +574,9 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_resource_count "${rendered_path}" HelmRelease observability coroot-operator 1
   assert_resource_count "${rendered_path}" Coroot observability coroot 0
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
-  assert_resource_count "${rendered_path}" HelmRelease monitoring kube-prometheus-stack 1
-  assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1
+    assert_resource_count "${rendered_path}" HelmRelease monitoring kube-prometheus-stack 1
+    assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1
+    assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 1
   assert_opencost_absent "${rendered_path}"
   assert_coroot_sso_controller_contract "${rendered_path}"
   assert_coroot_heartbeat_contract "${rendered_path}"

From a8c33b4bcc328a7a6daca12d50923f22951e8b08 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 22:02:21 +0200
Subject: [PATCH 05/26] test(observability): reject duplicate heartbeats

---
 scripts/validate-observability-option.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 04b6198..521a80d 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -114,11 +114,13 @@ assert_default_off() {
 
 assert_coroot_heartbeat_contract() {
   local rendered_path="$1"
+  local heartbeat_count
   local cronjob_contract
   local heartbeat_egress_contract
   local coroot_selector_contract
   local broad_external_egress
 
+  heartbeat_count="$(resource_count "${rendered_path}" CronJob observability cluster-heartbeat)"
   cronjob_contract="$(
     yq eval-all -o=json '.' "${rendered_path}" |
       jq -s '[.[] |
@@ -180,7 +182,7 @@ assert_coroot_heartbeat_contract() {
         select(any(.toEntities[]?; . == "world" or . == "all") or any(.toFQDNs[]?; has("matchPattern")))] | length'
   )"
 
-  if [[ "${cronjob_contract}" != "1" ]]; then
+  if [[ "${heartbeat_count}" != "1" || "${cronjob_contract}" != "1" ]]; then
     echo "Coroot profile must render one hardened five-minute cluster heartbeat in ${rendered_path}" >&2
     return 1
   fi

From 6ea9ae2b71847eed34172fa87415b1079c37172d Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 22:06:48 +0200
Subject: [PATCH 06/26] docs(observability): describe heartbeat validator

---
 scripts/validate-observability-option.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 521a80d..b5f8f6a 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -112,6 +112,8 @@ assert_default_off() {
   assert_resource_count "${rendered_path}" HTTPRoute observability coroot 0
 }
 
+# Validates the singleton workload, isolation selector, and exact egress contract
+# rendered by each explicit Coroot profile.
 assert_coroot_heartbeat_contract() {
   local rendered_path="$1"
   local heartbeat_count

From 3c6766a77a3fceb56c7f814cf9372a14d50d1af1 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 22:17:35 +0200
Subject: [PATCH 07/26] style(observability): align contract checks

---
 scripts/validate-observability-option.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index b5f8f6a..946180a 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -578,9 +578,9 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_resource_count "${rendered_path}" HelmRelease observability coroot-operator 1
   assert_resource_count "${rendered_path}" Coroot observability coroot 0
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
-    assert_resource_count "${rendered_path}" HelmRelease monitoring kube-prometheus-stack 1
-    assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1
-    assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 1
+  assert_resource_count "${rendered_path}" HelmRelease monitoring kube-prometheus-stack 1
+  assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1
+  assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 1
   assert_opencost_absent "${rendered_path}"
   assert_coroot_sso_controller_contract "${rendered_path}"
   assert_coroot_heartbeat_contract "${rendered_path}"

From 07cbf7f70e26358f4994102de8f33b8fa51250bd Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 22:27:00 +0200
Subject: [PATCH 08/26] fix(observability): preserve heartbeat signal

---
 docs/TEMPLATING.md                            | 13 ++--
 docs/dr/alerting.md                           | 12 ++--
 .../best-practices/add-default-deny.yaml      | 17 ++++-
 .../cilium-network-policy-heartbeat.yaml      |  4 +-
 .../coroot/cilium-network-policy.yaml         |  6 +-
 .../controllers/coroot/cron-job.yaml          |  2 +
 .../kustomization.yaml                        | 24 +++++++
 .../kustomization.yaml                        |  1 +
 .../kustomization.yaml                        |  1 +
 scripts/validate-observability-option.sh      | 63 +++++++++++++++++--
 10 files changed, 123 insertions(+), 20 deletions(-)
 create mode 100644 k8s/components/observability-coroot/disable-watchdog-heartbeat/kustomization.yaml

diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index ea460e1..799751c 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -90,10 +90,15 @@ Per-alert notifications remain visible only in the Coroot UI; this avoids sendin
 every noisy alert to the shared webhook. Local / Docker Coroot stays notification-free.
 The profile also stages one hardened `cluster-heartbeat` CronJob for the cluster
 dead-man signal. It reuses the existing encrypted heartbeat URL and does not add a new secret.
-Its policy permits only `hc-ping.com:443`; the invalid fallback keeps local and
-unconfigured instances inert. Kube-prometheus-stack keeps owning its alert rules,
-including Flux reconciliation alerts, until that remaining metrics and alerting
-migration is delivered separately.
+It runs at platform-critical priority, and its dedicated policy permits only an
+exact `hc-ping.com` DNS query plus `hc-ping.com:443`. The namespace deny and broad
+DNS policies exclude only the CronJob's purpose-specific label, so their explicit
+deny cannot override the narrower workload policy. The invalid fallback keeps
+local and unconfigured instances inert. The profile disables only Watchdog to
+prevent Alertmanager from refreshing the same external check and masking a broken
+CronJob. Kube-prometheus-stack keeps owning its alert rules except Watchdog,
+including Flux reconciliation alerts, until that remaining migration is delivered
+separately.
 
 Open the Coroot UI at `https://observability.<your-domain>`. Community Edition
 does not provide native OIDC, so this profile sends every UI request through the
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index c9a5939..80ff112 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -59,10 +59,14 @@ Slack out-of-band.
 
 The Coroot profile uses the dedicated `cluster-heartbeat` CronJob instead. It
 pings the same existing URL every five minutes, independently of Coroot, and
-its network policy allows only `hc-ping.com:443`. The invalid fallback keeps
-local and unconfigured instances quiet. Flux reconciliation alerting still depends on kube-prometheus-stack,
-so that stack remains transitional until a later slice replaces those rules
-before removing it.
+its network policy allows only an exact `hc-ping.com` DNS query plus
+`hc-ping.com:443`. The namespace deny and broad DNS policies exclude only this
+purpose-labelled workload. The Coroot profile disables Watchdog so Alertmanager
+cannot refresh the same external check and hide a failed CronJob. The invalid
+fallback keeps local and unconfigured instances quiet.
+Flux reconciliation alerting still depends on kube-prometheus-stack, so that stack
+remains
+transitional until a later slice replaces those rules before removing it.
 
 Recommended monitor: [healthchecks.io](https://healthchecks.io) (free,
 open-source, native Slack integration). Create a check with a ~5 min
diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml
index 0e5d9fd..ce87a2a 100644
--- a/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml
+++ b/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml
@@ -43,7 +43,14 @@ spec:
         namespace: "{{request.object.metadata.name}}"
         data:
           spec:
-            endpointSelector: {}
+            # This one purpose-specific label is handled by an exact workload
+            # policy. Keeping the exclusion here prevents the explicit deny
+            # from overriding that narrower allow-list without creating a
+            # generic opt-out from the namespace guardrail.
+            endpointSelector:
+              matchExpressions:
+                - key: platform-heartbeat
+                  operator: DoesNotExist
             enableDefaultDeny:
               ingress: true
               egress: true
@@ -73,7 +80,13 @@ spec:
         namespace: "{{request.object.metadata.name}}"
         data:
           spec:
-            endpointSelector: {}
+            # The heartbeat policy carries its own exact DNS rule. Excluding
+            # it here prevents the namespace-wide DNS allow from widening the
+            # workload's permitted query names.
+            endpointSelector:
+              matchExpressions:
+                - key: platform-heartbeat
+                  operator: DoesNotExist
             egress:
               - toEndpoints:
                   - matchLabels:
diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
index 0b20f80..6379efc 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
@@ -6,7 +6,7 @@ metadata:
 spec:
   endpointSelector:
     matchLabels:
-      app: cluster-heartbeat
+      platform-heartbeat: "true"
   egress:
     - toFQDNs:
         - matchName: hc-ping.com
@@ -27,4 +27,4 @@ spec:
               protocol: TCP
           rules:
             dns:
-              - matchPattern: "*"
+              - matchName: hc-ping.com
diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
index b066859..ad82850 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
@@ -6,10 +6,8 @@ metadata:
 spec:
   endpointSelector:
     matchExpressions:
-      - key: app
-        operator: NotIn
-        values:
-          - cluster-heartbeat
+      - key: platform-heartbeat
+        operator: DoesNotExist
   ingress:
     # Coroot, its bundled stores, its agents, and the audit forwarder exchange
     # telemetry inside this namespace.
diff --git a/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml b/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
index db8541c..d32a9a4 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
@@ -30,9 +30,11 @@ spec:
         metadata:
           labels:
             app: cluster-heartbeat
+            platform-heartbeat: "true"
         spec:
           restartPolicy: Never
           automountServiceAccountToken: false
+          priorityClassName: platform-critical
           securityContext:
             runAsNonRoot: true
             runAsUser: 65532
diff --git a/k8s/components/observability-coroot/disable-watchdog-heartbeat/kustomization.yaml b/k8s/components/observability-coroot/disable-watchdog-heartbeat/kustomization.yaml
new file mode 100644
index 0000000..ce58e67
--- /dev/null
+++ b/k8s/components/observability-coroot/disable-watchdog-heartbeat/kustomization.yaml
@@ -0,0 +1,24 @@
+---
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+# Coroot profiles use the dedicated cluster-heartbeat CronJob for the existing
+# external check. Disable only Watchdog so Alertmanager cannot refresh the same
+# URL and mask a broken CronJob; every other default alert stays enabled.
+patches:
+  - target:
+      group: helm.toolkit.fluxcd.io
+      version: v2
+      kind: HelmRelease
+      name: kube-prometheus-stack
+      namespace: monitoring
+    patch: |
+      apiVersion: helm.toolkit.fluxcd.io/v2
+      kind: HelmRelease
+      metadata:
+        name: kube-prometheus-stack
+        namespace: monitoring
+      spec:
+        values:
+          defaultRules:
+            disabled:
+              Watchdog: true
diff --git a/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml b/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
index 16a6418..fc65a8b 100644
--- a/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
+++ b/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
@@ -5,6 +5,7 @@ resources:
   - ../infrastructure/controllers/
   - ../../../bases/infrastructure/controllers/coroot/
 components:
+  - ../../../components/observability-coroot/disable-watchdog-heartbeat/
   - ../../../components/observability-coroot/remove-controllers-loki-alloy/
   - ../../../components/observability-coroot/remove-controller-opencost/
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
index 3688282..9d06b93 100644
--- a/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
+++ b/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
@@ -5,6 +5,7 @@ resources:
   - ../infrastructure/controllers/
   - ../../../bases/infrastructure/controllers/coroot/
 components:
+  - ../../../components/observability-coroot/disable-watchdog-heartbeat/
   - ../../../components/observability-coroot/remove-controllers-loki-alloy/
   - ../../../components/observability-coroot/remove-controller-opencost/
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 946180a..5455b41 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -134,8 +134,10 @@ assert_coroot_heartbeat_contract() {
         select(.spec.jobTemplate.spec.backoffLimit == 0 and .spec.jobTemplate.spec.activeDeadlineSeconds == 120) |
         .spec.jobTemplate.spec.template as $template |
         select($template.metadata.labels.app == "cluster-heartbeat") |
+        select($template.metadata.labels["platform-heartbeat"] == "true") |
         $template.spec as $pod |
         select($pod.restartPolicy == "Never" and $pod.automountServiceAccountToken == false) |
+        select($pod.priorityClassName == "platform-critical") |
         select($pod.securityContext.runAsNonRoot == true and $pod.securityContext.runAsUser == 65532) |
         select($pod.securityContext.seccompProfile.type == "RuntimeDefault") |
         select($pod.containers | length == 1) |
@@ -160,20 +162,20 @@ assert_coroot_heartbeat_contract() {
     yq eval-all -o=json '.' "${rendered_path}" |
       jq -s '[.[] |
         select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-cluster-heartbeat") |
-        select(.spec.endpointSelector.matchLabels == {"app":"cluster-heartbeat"}) |
+        select(.spec.endpointSelector.matchLabels == {"platform-heartbeat":"true"}) |
         select(.spec.egress | length == 2) |
         select(any(.spec.egress[];
           .toFQDNs == [{"matchName":"hc-ping.com"}] and
           .toPorts == [{"ports":[{"port":"443","protocol":"TCP"}]}])) |
         select(any(.spec.egress[];
           .toEndpoints == [{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}] and
-          .toPorts == [{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}],"rules":{"dns":[{"matchPattern":"*"}]}}]))] | length'
+          .toPorts == [{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}],"rules":{"dns":[{"matchName":"hc-ping.com"}]}}]))] | length'
   )"
   coroot_selector_contract="$(
     yq eval-all -o=json '.' "${rendered_path}" |
       jq -s '[.[] |
         select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-coroot") |
-        select(.spec.endpointSelector.matchExpressions == [{"key":"app","operator":"NotIn","values":["cluster-heartbeat"]}]) |
+        select(.spec.endpointSelector.matchExpressions == [{"key":"platform-heartbeat","operator":"DoesNotExist"}]) |
         select(all(.spec.egress[]?.toFQDNs[]?; .matchName != "hc-ping.com"))] | length'
   )"
   broad_external_egress="$(
@@ -194,6 +196,51 @@ assert_coroot_heartbeat_contract() {
   fi
 }
 
+# Validates that only explicit Coroot profiles replace the Watchdog heartbeat.
+assert_watchdog_disabled() {
+  local rendered_path="$1"
+  local expected="$2"
+  local actual
+
+  actual="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s -r '[.[] |
+        select(.kind == "HelmRelease" and .metadata.namespace == "monitoring" and .metadata.name == "kube-prometheus-stack") |
+        (.spec.values.defaultRules.disabled.Watchdog // false)] |
+        if length == 1 then .[0] else "invalid-count:\(length)" end'
+  )"
+  if [[ "${actual}" != "${expected}" ]]; then
+    echo "expected kube-prometheus-stack Watchdog disabled=${expected} in ${rendered_path}, found ${actual}" >&2
+    return 1
+  fi
+}
+
+# Validates that generated deny and DNS policies defer the dedicated heartbeat
+# label to its narrower workload policy without creating a generic bypass.
+assert_heartbeat_policy_exclusion() {
+  local rendered_path="$1"
+  local rule_name
+  local matches
+
+  for rule_name in generate-default-deny generate-allow-dns; do
+    matches="$(
+      yq eval-all -o=json '.' "${rendered_path}" |
+        jq -s --arg rule_name "${rule_name}" '[.[] |
+          select(.kind == "ClusterPolicy" and .metadata.name == "add-default-deny") |
+          .spec.rules[] |
+          select(.name == $rule_name) |
+          select(.generate.data.spec.endpointSelector.matchExpressions == [{
+            "key":"platform-heartbeat",
+            "operator":"DoesNotExist"
+          }])] | length'
+    )"
+    if [[ "${matches}" != "1" ]]; then
+      echo "add-default-deny/${rule_name} must exclude only the dedicated heartbeat label in ${rendered_path}" >&2
+      return 1
+    fi
+  done
+}
+
 assert_opencost_present() {
   local rendered_path="$1"
 
@@ -522,6 +569,7 @@ done
 # Only the explicit Coroot profiles retire it.
 for rendered_path in "${local_controllers_default}" "${prod_controllers_default}"; do
   assert_opencost_present "${rendered_path}"
+  assert_watchdog_disabled "${rendered_path}" false
 done
 for rendered_path in "${local_infrastructure_default}" "${prod_infrastructure_default}"; do
   assert_opencost_resources_absent "${rendered_path}"
@@ -531,6 +579,7 @@ for rendered_path in "${local_infrastructure_default}" "${prod_infrastructure_de
   assert_security_exception_namespace_count "${rendered_path}" infrastructure-privileged observability 0
   assert_security_exception_namespace_count "${rendered_path}" controller-rbac observability 0
   assert_security_exception_namespace_count "${rendered_path}" service-account-tokens observability 0
+  assert_heartbeat_policy_exclusion "${rendered_path}"
 done
 assert_opencost_reference_count "${local_apps_default}" 1
 assert_opencost_reference_count "${prod_apps_default}" 1
@@ -584,6 +633,7 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_opencost_absent "${rendered_path}"
   assert_coroot_sso_controller_contract "${rendered_path}"
   assert_coroot_heartbeat_contract "${rendered_path}"
+  assert_watchdog_disabled "${rendered_path}" true
 done
 
 for rendered_path in "${docker_infrastructure}" "${hetzner_infrastructure}"; do
@@ -594,6 +644,7 @@ for rendered_path in "${docker_infrastructure}" "${hetzner_infrastructure}"; do
   assert_security_exception_namespace_count "${rendered_path}" controller-rbac observability 1
   assert_security_exception_namespace_count "${rendered_path}" service-account-tokens observability 1
   assert_security_exception_namespace_count "${rendered_path}" health-probes observability 0
+  assert_heartbeat_policy_exclusion "${rendered_path}"
 done
 
 assert_resource_count "${docker_infrastructure}" HelmRelease observability coroot-operator 0
@@ -748,7 +799,10 @@ for documented_boundary in \
   "retires the legacy Loki and Alloy log path" \
   "stages one hardened \`cluster-heartbeat\` CronJob" \
   "reuses the existing encrypted heartbeat URL" \
-  "permits only \`hc-ping.com:443\`" \
+  "exact \`hc-ping.com\` DNS query" \
+  "\`hc-ping.com:443\`" \
+  "runs at platform-critical priority" \
+  "disables only Watchdog" \
   "does not add a new secret" \
   "keeps kube-prometheus-stack" \
   "Cost allocation is therefore unavailable" \
@@ -773,6 +827,7 @@ for documented_heartbeat_boundary in \
   "The default profile keeps the \`Watchdog\` alert" \
   "The Coroot profile uses the dedicated \`cluster-heartbeat\` CronJob" \
   "hc-ping.com:443" \
+  "disables Watchdog" \
   "Flux reconciliation alerting still depends on kube-prometheus-stack"; do
   if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then
     echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2

From 1e5199889f2d2b6ddc5bbac7ca74fc62d3b07fec Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 22:35:26 +0200
Subject: [PATCH 09/26] fix(observability): deny heartbeat ingress

---
 .../controllers/coroot/cilium-network-policy-heartbeat.yaml  | 5 +++++
 scripts/validate-observability-option.sh                     | 2 ++
 2 files changed, 7 insertions(+)

diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
index 6379efc..a3ae947 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
@@ -7,6 +7,11 @@ spec:
   endpointSelector:
     matchLabels:
       platform-heartbeat: "true"
+  enableDefaultDeny:
+    ingress: true
+    egress: true
+  ingressDeny:
+    - {}
   egress:
     - toFQDNs:
         - matchName: hc-ping.com
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 5455b41..99f87c1 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -163,6 +163,8 @@ assert_coroot_heartbeat_contract() {
       jq -s '[.[] |
         select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-cluster-heartbeat") |
         select(.spec.endpointSelector.matchLabels == {"platform-heartbeat":"true"}) |
+        select(.spec.enableDefaultDeny == {"ingress":true,"egress":true}) |
+        select(.spec.ingressDeny == [{}]) |
         select(.spec.egress | length == 2) |
         select(any(.spec.egress[];
           .toFQDNs == [{"matchName":"hc-ping.com"}] and

From 408f8359517faeb9d296b5b54acd28510680014b Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 23:05:32 +0200
Subject: [PATCH 10/26] fix(observability): scope heartbeat policy inputs

---
 .github/workflows/bootstrap.yaml              | 31 ++++++++-
 docs/BOOTSTRAP.md                             |  4 ++
 docs/TEMPLATING.md                            | 23 ++++---
 docs/dr/alerting.md                           | 12 ++--
 .../best-practices/add-default-deny.yaml      | 69 +++++++++++++++++--
 .../cilium-network-policy-heartbeat.yaml      |  4 +-
 .../variables-cluster-secret.enc.yaml         |  1 +
 scripts/validate-observability-option.sh      | 50 +++++++++++---
 8 files changed, 160 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml
index 094a1e2..b9870e6 100644
--- a/.github/workflows/bootstrap.yaml
+++ b/.github/workflows/bootstrap.yaml
@@ -190,9 +190,33 @@ jobs:
         run: |
           set -euo pipefail
           # Internal secrets that need no human input — generate random values.
-          export DEX_CLIENT_SECRET="$(openssl rand -hex 32)"
-          export FLUX_WEB_CLIENT_SECRET="$(openssl rand -hex 32)"
-          export OAUTH2_PROXY_COOKIE_SECRET="$(openssl rand -base64 32)"
+          DEX_CLIENT_SECRET="$(openssl rand -hex 32)"
+          FLUX_WEB_CLIENT_SECRET="$(openssl rand -hex 32)"
+          OAUTH2_PROXY_COOKIE_SECRET="$(openssl rand -base64 32)"
+          export DEX_CLIENT_SECRET FLUX_WEB_CLIENT_SECRET OAUTH2_PROXY_COOKIE_SECRET
+
+          # Derive the non-secret FQDN used by the Coroot heartbeat policy from
+          # the existing URL input. This keeps custom monitor providers working
+          # without adding another GitHub secret or operator-supplied value.
+          ALERTMANAGER_HEARTBEAT_HOST="hc-ping.com"
+          if [[ -n "${ALERTMANAGER_HEARTBEAT_URL:-}" ]]; then
+            if [[ "${ALERTMANAGER_HEARTBEAT_URL}" != https://* ]]; then
+              echo "ALERTMANAGER_HEARTBEAT_URL must use https://" >&2
+              exit 1
+            fi
+            heartbeat_host="${ALERTMANAGER_HEARTBEAT_URL#*://}"
+            heartbeat_host="${heartbeat_host%%/*}"
+            ALERTMANAGER_HEARTBEAT_HOST="${heartbeat_host%%:*}"
+            if [[ -z "${ALERTMANAGER_HEARTBEAT_HOST}" ||
+              "${ALERTMANAGER_HEARTBEAT_HOST}" == *[!A-Za-z0-9.-]* ||
+              "${ALERTMANAGER_HEARTBEAT_HOST}" == .* ||
+              "${ALERTMANAGER_HEARTBEAT_HOST}" == *. ||
+              "${ALERTMANAGER_HEARTBEAT_HOST}" == *..* ]]; then
+              echo "ALERTMANAGER_HEARTBEAT_URL must contain a valid DNS hostname" >&2
+              exit 1
+            fi
+          fi
+          export ALERTMANAGER_HEARTBEAT_HOST
 
           fill_base() { # variables-base-secret
             yq -i '
@@ -207,6 +231,7 @@ jobs:
           yq -i '
             .stringData.alertmanager_webhook_url = strenv(ALERTMANAGER_WEBHOOK_URL) |
             .stringData.alertmanager_heartbeat_url = strenv(ALERTMANAGER_HEARTBEAT_URL) |
+            .stringData.alertmanager_heartbeat_host = strenv(ALERTMANAGER_HEARTBEAT_HOST) |
             .stringData.dex_client_secret = strenv(DEX_CLIENT_SECRET) |
             .stringData.flux_web_client_secret = strenv(FLUX_WEB_CLIENT_SECRET) |
             .stringData.oauth2_proxy_cookie_secret = strenv(OAUTH2_PROXY_COOKIE_SECRET) |
diff --git a/docs/BOOTSTRAP.md b/docs/BOOTSTRAP.md
index f868851..1653526 100644
--- a/docs/BOOTSTRAP.md
+++ b/docs/BOOTSTRAP.md
@@ -115,6 +115,10 @@ Set these in your **new instance repository** under *Settings → Secrets and va
 | `GITHUB_APP_CLIENT_SECRET` | **Client secret** of the *cluster SSO* GitHub App (Dex’s GitHub connector) | The SSO App’s settings page (generate a client secret). |
 | `APP_PRIVATE_KEY` | Private key (`.pem`) of the **bootstrap** GitHub App | Generated when you created the bootstrap App. |
 
+Bootstrap validates the heartbeat URL and derives its DNS hostname into the same
+encrypted cluster Secret for the Coroot network policy; no separate hostname
+secret or input is required.
+
 ### Auto-generated by the bootstrap — you never set these
 
 | Secret | Scope | What it is |
diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index 799751c..b840236 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -90,15 +90,20 @@ Per-alert notifications remain visible only in the Coroot UI; this avoids sendin
 every noisy alert to the shared webhook. Local / Docker Coroot stays notification-free.
 The profile also stages one hardened `cluster-heartbeat` CronJob for the cluster
 dead-man signal. It reuses the existing encrypted heartbeat URL and does not add a new secret.
-It runs at platform-critical priority, and its dedicated policy permits only an
-exact `hc-ping.com` DNS query plus `hc-ping.com:443`. The namespace deny and broad
-DNS policies exclude only the CronJob's purpose-specific label, so their explicit
-deny cannot override the narrower workload policy. The invalid fallback keeps
-local and unconfigured instances inert. The profile disables only Watchdog to
-prevent Alertmanager from refreshing the same external check and masking a broken
-CronJob. Kube-prometheus-stack keeps owning its alert rules except Watchdog,
-including Flux reconciliation alerts, until that remaining migration is delivered
-separately.
+It runs at platform-critical priority, and its dedicated policy permits only the
+configured heartbeat hostname's exact DNS query plus HTTPS. During bootstrap the
+policy host is derived automatically from the existing encrypted URL and stored in
+that same Secret, so there is no new secret or GitHub input. Existing synced
+instances using a non-healthchecks provider should add the URL's hostname as
+`alertmanager_heartbeat_host` before selecting the Coroot profile; the fallback remains
+`hc-ping.com`. The namespace deny and broad DNS policies exclude the CronJob's
+purpose-specific label only inside `observability`, so their explicit deny cannot
+override the narrower workload policy or create a label-based bypass elsewhere.
+The invalid URL fallback keeps local and unconfigured instances inert. The profile
+disables only Watchdog to prevent Alertmanager from refreshing the same external
+check and masking a broken CronJob.
+Kube-prometheus-stack keeps owning its alert rules except Watchdog, including Flux
+reconciliation alerts, until that remaining migration is delivered separately.
 
 Open the Coroot UI at `https://observability.<your-domain>`. Community Edition
 does not provide native OIDC, so this profile sends every UI request through the
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index 80ff112..bb8fda6 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -59,11 +59,15 @@ Slack out-of-band.
 
 The Coroot profile uses the dedicated `cluster-heartbeat` CronJob instead. It
 pings the same existing URL every five minutes, independently of Coroot, and
-its network policy allows only an exact `hc-ping.com` DNS query plus
-`hc-ping.com:443`. The namespace deny and broad DNS policies exclude only this
-purpose-labelled workload. The Coroot profile disables Watchdog so Alertmanager
+its network policy allows only that URL's exact hostname for DNS and HTTPS.
+The policy host is derived automatically during bootstrap and stored beside the
+URL in the existing encrypted Secret. For an already-synced instance using a
+non-healthchecks provider, add the matching `alertmanager_heartbeat_host` before
+selecting the Coroot profile; otherwise the safe fallback is `hc-ping.com`.
+The namespace deny and broad DNS policies exclude this purpose-labelled workload
+only in `observability`. The Coroot profile disables Watchdog so Alertmanager
 cannot refresh the same external check and hide a failed CronJob. The invalid
-fallback keeps local and unconfigured instances quiet.
+URL fallback keeps local and unconfigured instances quiet.
 Flux reconciliation alerting still depends on kube-prometheus-stack, so that stack
 remains
 transitional until a later slice replaces those rules before removing it.
diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml
index ce87a2a..9df8d8e 100644
--- a/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml
+++ b/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml
@@ -34,6 +34,35 @@ spec:
                 - kube-system
                 - kube-public
                 - kube-node-lease
+                - observability
+      generate:
+        generateExisting: true
+        apiVersion: cilium.io/v2
+        kind: CiliumNetworkPolicy
+        name: default-deny
+        synchronize: true
+        namespace: "{{request.object.metadata.name}}"
+        data:
+          spec:
+            endpointSelector: {}
+            enableDefaultDeny:
+              ingress: true
+              egress: true
+            ingressDeny:
+              - {}
+            egressDeny:
+              - {}
+    # The observability namespace has one purpose-labelled CronJob with a
+    # narrower policy. Scope the exception to this namespace so the label can
+    # never become a repository-wide opt-out from default deny.
+    - name: generate-default-deny-observability
+      match:
+        any:
+          - resources:
+              kinds:
+                - Namespace
+              names:
+                - observability
       generate:
         generateExisting: true
         apiVersion: cilium.io/v2
@@ -43,10 +72,6 @@ spec:
         namespace: "{{request.object.metadata.name}}"
         data:
           spec:
-            # This one purpose-specific label is handled by an exact workload
-            # policy. Keeping the exclusion here prevents the explicit deny
-            # from overriding that narrower allow-list without creating a
-            # generic opt-out from the namespace guardrail.
             endpointSelector:
               matchExpressions:
                 - key: platform-heartbeat
@@ -71,6 +96,39 @@ spec:
                 - kube-system
                 - kube-public
                 - kube-node-lease
+                - observability
+      generate:
+        generateExisting: true
+        apiVersion: cilium.io/v2
+        kind: CiliumNetworkPolicy
+        name: allow-dns
+        synchronize: true
+        namespace: "{{request.object.metadata.name}}"
+        data:
+          spec:
+            endpointSelector: {}
+            egress:
+              - toEndpoints:
+                  - matchLabels:
+                      k8s:io.kubernetes.pod.namespace: kube-system
+                      k8s-app: kube-dns
+                toPorts:
+                  - ports:
+                      - port: "53"
+                        protocol: UDP
+                      - port: "53"
+                        protocol: TCP
+    # The heartbeat policy carries its own exact DNS rule. Excluding it only
+    # in observability prevents the broad namespace DNS allow from widening
+    # that workload without opening a label-based bypass elsewhere.
+    - name: generate-allow-dns-observability
+      match:
+        any:
+          - resources:
+              kinds:
+                - Namespace
+              names:
+                - observability
       generate:
         generateExisting: true
         apiVersion: cilium.io/v2
@@ -80,9 +138,6 @@ spec:
         namespace: "{{request.object.metadata.name}}"
         data:
           spec:
-            # The heartbeat policy carries its own exact DNS rule. Excluding
-            # it here prevents the namespace-wide DNS allow from widening the
-            # workload's permitted query names.
             endpointSelector:
               matchExpressions:
                 - key: platform-heartbeat
diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
index a3ae947..16ea917 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
@@ -14,7 +14,7 @@ spec:
     - {}
   egress:
     - toFQDNs:
-        - matchName: hc-ping.com
+        - matchName: ${alertmanager_heartbeat_host:=hc-ping.com}
       toPorts:
         - ports:
             - port: "443"
@@ -32,4 +32,4 @@ spec:
               protocol: TCP
           rules:
             dns:
-              - matchName: hc-ping.com
+              - matchName: ${alertmanager_heartbeat_host:=hc-ping.com}
diff --git a/k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml b/k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
index bd7b5ef..124cbe5 100644
--- a/k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
+++ b/k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
@@ -17,3 +17,4 @@ stringData:
   r2_access_key_id: REPLACE_ME
   r2_secret_access_key: REPLACE_ME
   alertmanager_heartbeat_url: https://example.com/REPLACE_ME
+  alertmanager_heartbeat_host: example.com
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 99f87c1..90e025e 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -167,11 +167,11 @@ assert_coroot_heartbeat_contract() {
         select(.spec.ingressDeny == [{}]) |
         select(.spec.egress | length == 2) |
         select(any(.spec.egress[];
-          .toFQDNs == [{"matchName":"hc-ping.com"}] and
+          .toFQDNs == [{"matchName":"${alertmanager_heartbeat_host:=hc-ping.com}"}] and
           .toPorts == [{"ports":[{"port":"443","protocol":"TCP"}]}])) |
         select(any(.spec.egress[];
           .toEndpoints == [{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}] and
-          .toPorts == [{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}],"rules":{"dns":[{"matchName":"hc-ping.com"}]}}]))] | length'
+          .toPorts == [{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}],"rules":{"dns":[{"matchName":"${alertmanager_heartbeat_host:=hc-ping.com}"}]}}]))] | length'
   )"
   coroot_selector_contract="$(
     yq eval-all -o=json '.' "${rendered_path}" |
@@ -193,7 +193,7 @@ assert_coroot_heartbeat_contract() {
     return 1
   fi
   if [[ "${heartbeat_egress_contract}" != "1" || "${coroot_selector_contract}" != "1" || "${broad_external_egress}" != "0" ]]; then
-    echo "Coroot heartbeat must be excluded from the shared policy and have one exact hc-ping.com:443 egress path without wildcard or world access in ${rendered_path}" >&2
+    echo "Coroot heartbeat must be excluded from broad policies and have one exact configured-host egress path without wildcard or world access in ${rendered_path}" >&2
     return 1
   fi
 }
@@ -217,11 +217,12 @@ assert_watchdog_disabled() {
   fi
 }
 
-# Validates that generated deny and DNS policies defer the dedicated heartbeat
-# label to its narrower workload policy without creating a generic bypass.
+# Validates that only the observability namespace's generated policies defer
+# the dedicated heartbeat label to its narrower workload policy.
 assert_heartbeat_policy_exclusion() {
   local rendered_path="$1"
   local rule_name
+  local scoped_rule_name
   local matches
 
   for rule_name in generate-default-deny generate-allow-dns; do
@@ -231,13 +232,30 @@ assert_heartbeat_policy_exclusion() {
           select(.kind == "ClusterPolicy" and .metadata.name == "add-default-deny") |
           .spec.rules[] |
           select(.name == $rule_name) |
+          select(.exclude.any[0].resources.names | index("observability")) |
+          select(.generate.data.spec.endpointSelector == {})] | length'
+    )"
+    if [[ "${matches}" != "1" ]]; then
+      echo "add-default-deny/${rule_name} must retain the global guardrail and exclude observability for its scoped replacement in ${rendered_path}" >&2
+      return 1
+    fi
+
+    scoped_rule_name="${rule_name}-observability"
+    matches="$(
+      yq eval-all -o=json '.' "${rendered_path}" |
+        jq -s --arg rule_name "${scoped_rule_name}" '[.[] |
+          select(.kind == "ClusterPolicy" and .metadata.name == "add-default-deny") |
+          .spec.rules[] |
+          select(.name == $rule_name) |
+          select(.match.any[0].resources.kinds == ["Namespace"]) |
+          select(.match.any[0].resources.names == ["observability"]) |
           select(.generate.data.spec.endpointSelector.matchExpressions == [{
             "key":"platform-heartbeat",
             "operator":"DoesNotExist"
           }])] | length'
     )"
     if [[ "${matches}" != "1" ]]; then
-      echo "add-default-deny/${rule_name} must exclude only the dedicated heartbeat label in ${rendered_path}" >&2
+      echo "add-default-deny/${scoped_rule_name} must scope the heartbeat exclusion to observability in ${rendered_path}" >&2
       return 1
     fi
   done
@@ -801,10 +819,11 @@ for documented_boundary in \
   "retires the legacy Loki and Alloy log path" \
   "stages one hardened \`cluster-heartbeat\` CronJob" \
   "reuses the existing encrypted heartbeat URL" \
-  "exact \`hc-ping.com\` DNS query" \
-  "\`hc-ping.com:443\`" \
+  "configured heartbeat hostname's exact DNS query" \
+  "\`alertmanager_heartbeat_host\`" \
   "runs at platform-critical priority" \
   "disables only Watchdog" \
+  "host is derived automatically" \
   "does not add a new secret" \
   "keeps kube-prometheus-stack" \
   "Cost allocation is therefore unavailable" \
@@ -824,11 +843,24 @@ for documented_boundary in \
   fi
 done
 
+bootstrap_workflow="${repo_root}/.github/workflows/bootstrap.yaml"
+for heartbeat_host_boundary in \
+  "heartbeat_host=\"\${ALERTMANAGER_HEARTBEAT_URL#*://}\"" \
+  "heartbeat_host=\"\${heartbeat_host%%/*}\"" \
+  "ALERTMANAGER_HEARTBEAT_HOST=\"\${heartbeat_host%%:*}\"" \
+  '.stringData.alertmanager_heartbeat_host = strenv(ALERTMANAGER_HEARTBEAT_HOST)'; do
+  if ! grep -Fq "${heartbeat_host_boundary}" "${bootstrap_workflow}"; then
+    echo "bootstrap workflow does not derive the heartbeat policy host from the configured URL: ${heartbeat_host_boundary}" >&2
+    exit 1
+  fi
+done
+
 alerting_guide="${repo_root}/docs/dr/alerting.md"
 for documented_heartbeat_boundary in \
   "The default profile keeps the \`Watchdog\` alert" \
   "The Coroot profile uses the dedicated \`cluster-heartbeat\` CronJob" \
-  "hc-ping.com:443" \
+  "policy host is derived automatically" \
+  "alertmanager_heartbeat_host" \
   "disables Watchdog" \
   "Flux reconciliation alerting still depends on kube-prometheus-stack"; do
   if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then

From 934334058ce6bbee2d8226d73eb1ec4818c16d0a Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 23:19:59 +0200
Subject: [PATCH 11/26] fix(observability): validate heartbeat port

---
 .github/workflows/bootstrap.yaml         | 13 ++++++++++---
 docs/BOOTSTRAP.md                        |  6 +++---
 docs/TEMPLATING.md                       |  4 +++-
 docs/dr/alerting.md                      |  2 ++
 scripts/validate-observability-option.sh |  9 ++++++---
 5 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml
index b9870e6..a85d2cf 100644
--- a/.github/workflows/bootstrap.yaml
+++ b/.github/workflows/bootstrap.yaml
@@ -204,9 +204,16 @@ jobs:
               echo "ALERTMANAGER_HEARTBEAT_URL must use https://" >&2
               exit 1
             fi
-            heartbeat_host="${ALERTMANAGER_HEARTBEAT_URL#*://}"
-            heartbeat_host="${heartbeat_host%%/*}"
-            ALERTMANAGER_HEARTBEAT_HOST="${heartbeat_host%%:*}"
+            heartbeat_authority="${ALERTMANAGER_HEARTBEAT_URL#*://}"
+            heartbeat_authority="${heartbeat_authority%%/*}"
+            if [[ "${heartbeat_authority}" == *:* ]]; then
+              heartbeat_port="${heartbeat_authority##*:}"
+              if [[ "${heartbeat_port}" != "443" ]]; then
+                echo "ALERTMANAGER_HEARTBEAT_URL must use HTTPS port 443" >&2
+                exit 1
+              fi
+            fi
+            ALERTMANAGER_HEARTBEAT_HOST="${heartbeat_authority%%:*}"
             if [[ -z "${ALERTMANAGER_HEARTBEAT_HOST}" ||
               "${ALERTMANAGER_HEARTBEAT_HOST}" == *[!A-Za-z0-9.-]* ||
               "${ALERTMANAGER_HEARTBEAT_HOST}" == .* ||
diff --git a/docs/BOOTSTRAP.md b/docs/BOOTSTRAP.md
index 1653526..a7fbe3d 100644
--- a/docs/BOOTSTRAP.md
+++ b/docs/BOOTSTRAP.md
@@ -115,9 +115,9 @@ Set these in your **new instance repository** under *Settings → Secrets and va
 | `GITHUB_APP_CLIENT_SECRET` | **Client secret** of the *cluster SSO* GitHub App (Dex’s GitHub connector) | The SSO App’s settings page (generate a client secret). |
 | `APP_PRIVATE_KEY` | Private key (`.pem`) of the **bootstrap** GitHub App | Generated when you created the bootstrap App. |
 
-Bootstrap validates the heartbeat URL and derives its DNS hostname into the same
-encrypted cluster Secret for the Coroot network policy; no separate hostname
-secret or input is required.
+Bootstrap validates that the heartbeat URL uses HTTPS port 443 and derives its
+DNS hostname into the same encrypted cluster Secret for the Coroot network
+policy; no separate hostname secret or input is required.
 
 ### Auto-generated by the bootstrap — you never set these
 
diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index b840236..a0a2aea 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -96,7 +96,9 @@ policy host is derived automatically from the existing encrypted URL and stored
 that same Secret, so there is no new secret or GitHub input. Existing synced
 instances using a non-healthchecks provider should add the URL's hostname as
 `alertmanager_heartbeat_host` before selecting the Coroot profile; the fallback remains
-`hc-ping.com`. The namespace deny and broad DNS policies exclude the CronJob's
+`hc-ping.com`. Custom providers must accept HTTPS on port 443, which bootstrap
+validates because the narrow policy deliberately permits no other port. The
+namespace deny and broad DNS policies exclude the CronJob's
 purpose-specific label only inside `observability`, so their explicit deny cannot
 override the narrower workload policy or create a label-based bypass elsewhere.
 The invalid URL fallback keeps local and unconfigured instances inert. The profile
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index bb8fda6..bf65f19 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -64,6 +64,8 @@ The policy host is derived automatically during bootstrap and stored beside the
 URL in the existing encrypted Secret. For an already-synced instance using a
 non-healthchecks provider, add the matching `alertmanager_heartbeat_host` before
 selecting the Coroot profile; otherwise the safe fallback is `hc-ping.com`.
+Custom providers must accept HTTPS on port 443; bootstrap rejects other ports
+because the narrow heartbeat policy deliberately permits no alternative port.
 The namespace deny and broad DNS policies exclude this purpose-labelled workload
 only in `observability`. The Coroot profile disables Watchdog so Alertmanager
 cannot refresh the same external check and hide a failed CronJob. The invalid
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 90e025e..e310be9 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -824,6 +824,7 @@ for documented_boundary in \
   "runs at platform-critical priority" \
   "disables only Watchdog" \
   "host is derived automatically" \
+  "Custom providers must accept HTTPS on port 443" \
   "does not add a new secret" \
   "keeps kube-prometheus-stack" \
   "Cost allocation is therefore unavailable" \
@@ -845,9 +846,11 @@ done
 
 bootstrap_workflow="${repo_root}/.github/workflows/bootstrap.yaml"
 for heartbeat_host_boundary in \
-  "heartbeat_host=\"\${ALERTMANAGER_HEARTBEAT_URL#*://}\"" \
-  "heartbeat_host=\"\${heartbeat_host%%/*}\"" \
-  "ALERTMANAGER_HEARTBEAT_HOST=\"\${heartbeat_host%%:*}\"" \
+  "heartbeat_authority=\"\${ALERTMANAGER_HEARTBEAT_URL#*://}\"" \
+  "heartbeat_authority=\"\${heartbeat_authority%%/*}\"" \
+  "heartbeat_port=\"\${heartbeat_authority##*:}\"" \
+  "[[ \"\${heartbeat_port}\" != \"443\" ]]" \
+  "ALERTMANAGER_HEARTBEAT_HOST=\"\${heartbeat_authority%%:*}\"" \
   '.stringData.alertmanager_heartbeat_host = strenv(ALERTMANAGER_HEARTBEAT_HOST)'; do
   if ! grep -Fq "${heartbeat_host_boundary}" "${bootstrap_workflow}"; then
     echo "bootstrap workflow does not derive the heartbeat policy host from the configured URL: ${heartbeat_host_boundary}" >&2

From ba5d1a1ce12afc6f7eef09a960a12f51db11e964 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 23:25:18 +0200
Subject: [PATCH 12/26] docs(observability): document shell helpers

---
 .github/workflows/bootstrap.yaml         | 3 ++-
 scripts/validate-observability-option.sh | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml
index a85d2cf..015fe53 100644
--- a/.github/workflows/bootstrap.yaml
+++ b/.github/workflows/bootstrap.yaml
@@ -225,7 +225,8 @@ jobs:
           fi
           export ALERTMANAGER_HEARTBEAT_HOST
 
-          fill_base() { # variables-base-secret
+          # Populates the base Secret fields shared by every generated instance.
+          fill_base() {
             yq -i '
               .stringData.cloudflare_api_token = strenv(CLOUDFLARE_API_TOKEN) |
               .stringData.r2_access_key_id = strenv(R2_ACCESS_KEY_ID) |
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index e310be9..38c4e29 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -100,6 +100,7 @@ assert_flux_path() {
   fi
 }
 
+# Verifies that the default profiles render no Coroot-owned resources.
 assert_default_off() {
   local rendered_path="$1"
 

From 2e190f8be6dadbfd1918da7af1f87b1914483520 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 23:40:31 +0200
Subject: [PATCH 13/26] fix(observability): preserve staged heartbeat safety

---
 .github/workflows/bootstrap.yaml              |  41 +------
 docs/BOOTSTRAP.md                             |   4 -
 docs/TEMPLATING.md                            |  26 ++--
 docs/dr/alerting.md                           |  24 ++--
 .../best-practices/add-default-deny.yaml      |  68 ----------
 .../cilium-network-policy-heartbeat.yaml      |   4 +-
 .../variables-cluster-secret.enc.yaml         |   1 -
 .../kustomization.yaml                        |  24 ----
 .../isolate-heartbeat/kustomization.yaml      |  96 +++++++++++++++
 .../kustomization.yaml                        |   1 -
 .../infrastructure-coroot/kustomization.yaml  |   1 +
 .../kustomization.yaml                        |   1 -
 .../infrastructure-coroot/kustomization.yaml  |   1 +
 scripts/validate-observability-option.sh      | 116 ++++++++++++++----
 14 files changed, 213 insertions(+), 195 deletions(-)
 delete mode 100644 k8s/components/observability-coroot/disable-watchdog-heartbeat/kustomization.yaml
 create mode 100644 k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml

diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml
index 015fe53..094a1e2 100644
--- a/.github/workflows/bootstrap.yaml
+++ b/.github/workflows/bootstrap.yaml
@@ -190,43 +190,11 @@ jobs:
         run: |
           set -euo pipefail
           # Internal secrets that need no human input — generate random values.
-          DEX_CLIENT_SECRET="$(openssl rand -hex 32)"
-          FLUX_WEB_CLIENT_SECRET="$(openssl rand -hex 32)"
-          OAUTH2_PROXY_COOKIE_SECRET="$(openssl rand -base64 32)"
-          export DEX_CLIENT_SECRET FLUX_WEB_CLIENT_SECRET OAUTH2_PROXY_COOKIE_SECRET
+          export DEX_CLIENT_SECRET="$(openssl rand -hex 32)"
+          export FLUX_WEB_CLIENT_SECRET="$(openssl rand -hex 32)"
+          export OAUTH2_PROXY_COOKIE_SECRET="$(openssl rand -base64 32)"
 
-          # Derive the non-secret FQDN used by the Coroot heartbeat policy from
-          # the existing URL input. This keeps custom monitor providers working
-          # without adding another GitHub secret or operator-supplied value.
-          ALERTMANAGER_HEARTBEAT_HOST="hc-ping.com"
-          if [[ -n "${ALERTMANAGER_HEARTBEAT_URL:-}" ]]; then
-            if [[ "${ALERTMANAGER_HEARTBEAT_URL}" != https://* ]]; then
-              echo "ALERTMANAGER_HEARTBEAT_URL must use https://" >&2
-              exit 1
-            fi
-            heartbeat_authority="${ALERTMANAGER_HEARTBEAT_URL#*://}"
-            heartbeat_authority="${heartbeat_authority%%/*}"
-            if [[ "${heartbeat_authority}" == *:* ]]; then
-              heartbeat_port="${heartbeat_authority##*:}"
-              if [[ "${heartbeat_port}" != "443" ]]; then
-                echo "ALERTMANAGER_HEARTBEAT_URL must use HTTPS port 443" >&2
-                exit 1
-              fi
-            fi
-            ALERTMANAGER_HEARTBEAT_HOST="${heartbeat_authority%%:*}"
-            if [[ -z "${ALERTMANAGER_HEARTBEAT_HOST}" ||
-              "${ALERTMANAGER_HEARTBEAT_HOST}" == *[!A-Za-z0-9.-]* ||
-              "${ALERTMANAGER_HEARTBEAT_HOST}" == .* ||
-              "${ALERTMANAGER_HEARTBEAT_HOST}" == *. ||
-              "${ALERTMANAGER_HEARTBEAT_HOST}" == *..* ]]; then
-              echo "ALERTMANAGER_HEARTBEAT_URL must contain a valid DNS hostname" >&2
-              exit 1
-            fi
-          fi
-          export ALERTMANAGER_HEARTBEAT_HOST
-
-          # Populates the base Secret fields shared by every generated instance.
-          fill_base() {
+          fill_base() { # variables-base-secret
             yq -i '
               .stringData.cloudflare_api_token = strenv(CLOUDFLARE_API_TOKEN) |
               .stringData.r2_access_key_id = strenv(R2_ACCESS_KEY_ID) |
@@ -239,7 +207,6 @@ jobs:
           yq -i '
             .stringData.alertmanager_webhook_url = strenv(ALERTMANAGER_WEBHOOK_URL) |
             .stringData.alertmanager_heartbeat_url = strenv(ALERTMANAGER_HEARTBEAT_URL) |
-            .stringData.alertmanager_heartbeat_host = strenv(ALERTMANAGER_HEARTBEAT_HOST) |
             .stringData.dex_client_secret = strenv(DEX_CLIENT_SECRET) |
             .stringData.flux_web_client_secret = strenv(FLUX_WEB_CLIENT_SECRET) |
             .stringData.oauth2_proxy_cookie_secret = strenv(OAUTH2_PROXY_COOKIE_SECRET) |
diff --git a/docs/BOOTSTRAP.md b/docs/BOOTSTRAP.md
index a7fbe3d..f868851 100644
--- a/docs/BOOTSTRAP.md
+++ b/docs/BOOTSTRAP.md
@@ -115,10 +115,6 @@ Set these in your **new instance repository** under *Settings → Secrets and va
 | `GITHUB_APP_CLIENT_SECRET` | **Client secret** of the *cluster SSO* GitHub App (Dex’s GitHub connector) | The SSO App’s settings page (generate a client secret). |
 | `APP_PRIVATE_KEY` | Private key (`.pem`) of the **bootstrap** GitHub App | Generated when you created the bootstrap App. |
 
-Bootstrap validates that the heartbeat URL uses HTTPS port 443 and derives its
-DNS hostname into the same encrypted cluster Secret for the Coroot network
-policy; no separate hostname secret or input is required.
-
 ### Auto-generated by the bootstrap — you never set these
 
 | Secret | Scope | What it is |
diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index a0a2aea..34bfdb2 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -90,22 +90,16 @@ Per-alert notifications remain visible only in the Coroot UI; this avoids sendin
 every noisy alert to the shared webhook. Local / Docker Coroot stays notification-free.
 The profile also stages one hardened `cluster-heartbeat` CronJob for the cluster
 dead-man signal. It reuses the existing encrypted heartbeat URL and does not add a new secret.
-It runs at platform-critical priority, and its dedicated policy permits only the
-configured heartbeat hostname's exact DNS query plus HTTPS. During bootstrap the
-policy host is derived automatically from the existing encrypted URL and stored in
-that same Secret, so there is no new secret or GitHub input. Existing synced
-instances using a non-healthchecks provider should add the URL's hostname as
-`alertmanager_heartbeat_host` before selecting the Coroot profile; the fallback remains
-`hc-ping.com`. Custom providers must accept HTTPS on port 443, which bootstrap
-validates because the narrow policy deliberately permits no other port. The
-namespace deny and broad DNS policies exclude the CronJob's
-purpose-specific label only inside `observability`, so their explicit deny cannot
-override the narrower workload policy or create a label-based bypass elsewhere.
-The invalid URL fallback keeps local and unconfigured instances inert. The profile
-disables only Watchdog to prevent Alertmanager from refreshing the same external
-check and masking a broken CronJob.
-Kube-prometheus-stack keeps owning its alert rules except Watchdog, including Flux
-reconciliation alerts, until that remaining migration is delivered separately.
+It runs at platform-critical priority, and its dedicated policy permits only `hc-ping.com:443`
+plus that hostname's exact DNS query. The Coroot infrastructure
+overlay alone narrows the generated namespace policies around the CronJob's
+purpose-specific label, so default profiles retain their global selectors. The
+invalid URL fallback keeps local and unconfigured instances inert. During this
+staging slice the CronJob runs alongside Watchdog: kube-prometheus-stack keeps the
+existing dead-man path so already-synced custom-provider heartbeats cannot be lost
+before a later, independently validated cutover. Kube-prometheus-stack keeps owning its alert rules,
+including Flux reconciliation alerts, until that
+remaining migration is delivered separately.
 
 Open the Coroot UI at `https://observability.<your-domain>`. Community Edition
 does not provide native OIDC, so this profile sends every UI request through the
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index bf65f19..ade6d47 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -57,22 +57,16 @@ tight cadence (`repeat_interval: 50s`). If the cluster — or the Prometheus →
 Alertmanager pipeline — dies, the monitor stops receiving pings and notifies
 Slack out-of-band.
 
-The Coroot profile uses the dedicated `cluster-heartbeat` CronJob instead. It
-pings the same existing URL every five minutes, independently of Coroot, and
-its network policy allows only that URL's exact hostname for DNS and HTTPS.
-The policy host is derived automatically during bootstrap and stored beside the
-URL in the existing encrypted Secret. For an already-synced instance using a
-non-healthchecks provider, add the matching `alertmanager_heartbeat_host` before
-selecting the Coroot profile; otherwise the safe fallback is `hc-ping.com`.
-Custom providers must accept HTTPS on port 443; bootstrap rejects other ports
-because the narrow heartbeat policy deliberately permits no alternative port.
-The namespace deny and broad DNS policies exclude this purpose-labelled workload
-only in `observability`. The Coroot profile disables Watchdog so Alertmanager
-cannot refresh the same external check and hide a failed CronJob. The invalid
-URL fallback keeps local and unconfigured instances quiet.
+The Coroot profile uses the dedicated `cluster-heartbeat` CronJob alongside that
+path. It pings the same existing URL every five minutes, independently of Coroot,
+and its network policy allows only `hc-ping.com:443` plus that hostname's exact
+DNS query. The Coroot overlay alone narrows the namespace deny and DNS policies
+around this purpose-labelled workload; default profiles keep their global
+selectors. The invalid URL fallback keeps local and unconfigured instances quiet.
+Coroot keeps Watchdog during this staging slice so custom-provider heartbeats remain
+protected before a later, independently validated cutover.
 Flux reconciliation alerting still depends on kube-prometheus-stack, so that stack
-remains
-transitional until a later slice replaces those rules before removing it.
+remains transitional until a later slice replaces those rules before removing it.
 
 Recommended monitor: [healthchecks.io](https://healthchecks.io) (free,
 open-source, native Slack integration). Create a check with a ~5 min
diff --git a/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml b/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml
index 9df8d8e..0e5d9fd 100644
--- a/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml
+++ b/k8s/bases/infrastructure/cluster-policies/best-practices/add-default-deny.yaml
@@ -34,7 +34,6 @@ spec:
                 - kube-system
                 - kube-public
                 - kube-node-lease
-                - observability
       generate:
         generateExisting: true
         apiVersion: cilium.io/v2
@@ -52,37 +51,6 @@ spec:
               - {}
             egressDeny:
               - {}
-    # The observability namespace has one purpose-labelled CronJob with a
-    # narrower policy. Scope the exception to this namespace so the label can
-    # never become a repository-wide opt-out from default deny.
-    - name: generate-default-deny-observability
-      match:
-        any:
-          - resources:
-              kinds:
-                - Namespace
-              names:
-                - observability
-      generate:
-        generateExisting: true
-        apiVersion: cilium.io/v2
-        kind: CiliumNetworkPolicy
-        name: default-deny
-        synchronize: true
-        namespace: "{{request.object.metadata.name}}"
-        data:
-          spec:
-            endpointSelector:
-              matchExpressions:
-                - key: platform-heartbeat
-                  operator: DoesNotExist
-            enableDefaultDeny:
-              ingress: true
-              egress: true
-            ingressDeny:
-              - {}
-            egressDeny:
-              - {}
     - name: generate-allow-dns
       match:
         any:
@@ -96,7 +64,6 @@ spec:
                 - kube-system
                 - kube-public
                 - kube-node-lease
-                - observability
       generate:
         generateExisting: true
         apiVersion: cilium.io/v2
@@ -118,38 +85,3 @@ spec:
                         protocol: UDP
                       - port: "53"
                         protocol: TCP
-    # The heartbeat policy carries its own exact DNS rule. Excluding it only
-    # in observability prevents the broad namespace DNS allow from widening
-    # that workload without opening a label-based bypass elsewhere.
-    - name: generate-allow-dns-observability
-      match:
-        any:
-          - resources:
-              kinds:
-                - Namespace
-              names:
-                - observability
-      generate:
-        generateExisting: true
-        apiVersion: cilium.io/v2
-        kind: CiliumNetworkPolicy
-        name: allow-dns
-        synchronize: true
-        namespace: "{{request.object.metadata.name}}"
-        data:
-          spec:
-            endpointSelector:
-              matchExpressions:
-                - key: platform-heartbeat
-                  operator: DoesNotExist
-            egress:
-              - toEndpoints:
-                  - matchLabels:
-                      k8s:io.kubernetes.pod.namespace: kube-system
-                      k8s-app: kube-dns
-                toPorts:
-                  - ports:
-                      - port: "53"
-                        protocol: UDP
-                      - port: "53"
-                        protocol: TCP
diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
index 16ea917..a3ae947 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
@@ -14,7 +14,7 @@ spec:
     - {}
   egress:
     - toFQDNs:
-        - matchName: ${alertmanager_heartbeat_host:=hc-ping.com}
+        - matchName: hc-ping.com
       toPorts:
         - ports:
             - port: "443"
@@ -32,4 +32,4 @@ spec:
               protocol: TCP
           rules:
             dns:
-              - matchName: ${alertmanager_heartbeat_host:=hc-ping.com}
+              - matchName: hc-ping.com
diff --git a/k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml b/k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
index 124cbe5..bd7b5ef 100644
--- a/k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
+++ b/k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
@@ -17,4 +17,3 @@ stringData:
   r2_access_key_id: REPLACE_ME
   r2_secret_access_key: REPLACE_ME
   alertmanager_heartbeat_url: https://example.com/REPLACE_ME
-  alertmanager_heartbeat_host: example.com
diff --git a/k8s/components/observability-coroot/disable-watchdog-heartbeat/kustomization.yaml b/k8s/components/observability-coroot/disable-watchdog-heartbeat/kustomization.yaml
deleted file mode 100644
index ce58e67..0000000
--- a/k8s/components/observability-coroot/disable-watchdog-heartbeat/kustomization.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
----
-apiVersion: kustomize.config.k8s.io/v1alpha1
-kind: Component
-# Coroot profiles use the dedicated cluster-heartbeat CronJob for the existing
-# external check. Disable only Watchdog so Alertmanager cannot refresh the same
-# URL and mask a broken CronJob; every other default alert stays enabled.
-patches:
-  - target:
-      group: helm.toolkit.fluxcd.io
-      version: v2
-      kind: HelmRelease
-      name: kube-prometheus-stack
-      namespace: monitoring
-    patch: |
-      apiVersion: helm.toolkit.fluxcd.io/v2
-      kind: HelmRelease
-      metadata:
-        name: kube-prometheus-stack
-        namespace: monitoring
-      spec:
-        values:
-          defaultRules:
-            disabled:
-              Watchdog: true
diff --git a/k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml b/k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml
new file mode 100644
index 0000000..b42c58c
--- /dev/null
+++ b/k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml
@@ -0,0 +1,96 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+# Only Coroot profiles stage the purpose-labelled heartbeat. Exclude that label
+# from the generated namespace policies here, then replace both boundaries with
+# narrower policies so default profiles keep their global zero-trust selectors.
+patches:
+  - target:
+      group: kyverno.io
+      version: v1
+      kind: ClusterPolicy
+      name: add-default-deny
+    patch: |
+      - op: test
+        path: /spec/rules/0/name
+        value: generate-default-deny
+      - op: test
+        path: /spec/rules/0/exclude/any/0/resources/names/2
+        value: kube-node-lease
+      - op: add
+        path: /spec/rules/0/exclude/any/0/resources/names/-
+        value: observability
+      - op: test
+        path: /spec/rules/1/name
+        value: generate-allow-dns
+      - op: test
+        path: /spec/rules/1/exclude/any/0/resources/names/2
+        value: kube-node-lease
+      - op: add
+        path: /spec/rules/1/exclude/any/0/resources/names/-
+        value: observability
+      - op: add
+        path: /spec/rules/-
+        value:
+          name: generate-default-deny-observability
+          match:
+            any:
+              - resources:
+                  kinds:
+                    - Namespace
+                  names:
+                    - observability
+          generate:
+            generateExisting: true
+            apiVersion: cilium.io/v2
+            kind: CiliumNetworkPolicy
+            name: default-deny
+            synchronize: true
+            namespace: "{{request.object.metadata.name}}"
+            data:
+              spec:
+                endpointSelector:
+                  matchExpressions:
+                    - key: platform-heartbeat
+                      operator: DoesNotExist
+                enableDefaultDeny:
+                  ingress: true
+                  egress: true
+                ingressDeny:
+                  - {}
+                egressDeny:
+                  - {}
+      - op: add
+        path: /spec/rules/-
+        value:
+          name: generate-allow-dns-observability
+          match:
+            any:
+              - resources:
+                  kinds:
+                    - Namespace
+                  names:
+                    - observability
+          generate:
+            generateExisting: true
+            apiVersion: cilium.io/v2
+            kind: CiliumNetworkPolicy
+            name: allow-dns
+            synchronize: true
+            namespace: "{{request.object.metadata.name}}"
+            data:
+              spec:
+                endpointSelector:
+                  matchExpressions:
+                    - key: platform-heartbeat
+                      operator: DoesNotExist
+                egress:
+                  - toEndpoints:
+                      - matchLabels:
+                          k8s:io.kubernetes.pod.namespace: kube-system
+                          k8s-app: kube-dns
+                    toPorts:
+                      - ports:
+                          - port: "53"
+                            protocol: UDP
+                          - port: "53"
+                            protocol: TCP
diff --git a/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml b/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
index fc65a8b..16a6418 100644
--- a/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
+++ b/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
@@ -5,7 +5,6 @@ resources:
   - ../infrastructure/controllers/
   - ../../../bases/infrastructure/controllers/coroot/
 components:
-  - ../../../components/observability-coroot/disable-watchdog-heartbeat/
   - ../../../components/observability-coroot/remove-controllers-loki-alloy/
   - ../../../components/observability-coroot/remove-controller-opencost/
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/k8s/providers/docker/infrastructure-coroot/kustomization.yaml b/k8s/providers/docker/infrastructure-coroot/kustomization.yaml
index 42b0858..4753d03 100644
--- a/k8s/providers/docker/infrastructure-coroot/kustomization.yaml
+++ b/k8s/providers/docker/infrastructure-coroot/kustomization.yaml
@@ -6,4 +6,5 @@ resources:
   - ../../../bases/infrastructure/coroot/
 components:
   - ../../../components/observability-coroot/allow-host-access/
+  - ../../../components/observability-coroot/isolate-heartbeat/
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
index 9d06b93..3688282 100644
--- a/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
+++ b/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
@@ -5,7 +5,6 @@ resources:
   - ../infrastructure/controllers/
   - ../../../bases/infrastructure/controllers/coroot/
 components:
-  - ../../../components/observability-coroot/disable-watchdog-heartbeat/
   - ../../../components/observability-coroot/remove-controllers-loki-alloy/
   - ../../../components/observability-coroot/remove-controller-opencost/
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml b/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml
index 3b4c311..08c7cbd 100644
--- a/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml
+++ b/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml
@@ -7,6 +7,7 @@ resources:
   - ../../../bases/infrastructure/audit-log-forwarder/
 components:
   - ../../../components/observability-coroot/allow-host-access/
+  - ../../../components/observability-coroot/isolate-heartbeat/
   - ../../../components/observability-coroot/remove-opencost-references/
 patches:
   - path: patches/add-webhook-alerting.yaml
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 38c4e29..7ed6f33 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -168,11 +168,11 @@ assert_coroot_heartbeat_contract() {
         select(.spec.ingressDeny == [{}]) |
         select(.spec.egress | length == 2) |
         select(any(.spec.egress[];
-          .toFQDNs == [{"matchName":"${alertmanager_heartbeat_host:=hc-ping.com}"}] and
+          .toFQDNs == [{"matchName":"hc-ping.com"}] and
           .toPorts == [{"ports":[{"port":"443","protocol":"TCP"}]}])) |
         select(any(.spec.egress[];
           .toEndpoints == [{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}] and
-          .toPorts == [{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}],"rules":{"dns":[{"matchName":"${alertmanager_heartbeat_host:=hc-ping.com}"}]}}]))] | length'
+          .toPorts == [{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}],"rules":{"dns":[{"matchName":"hc-ping.com"}]}}]))] | length'
   )"
   coroot_selector_contract="$(
     yq eval-all -o=json '.' "${rendered_path}" |
@@ -199,7 +199,43 @@ assert_coroot_heartbeat_contract() {
   fi
 }
 
-# Validates that only explicit Coroot profiles replace the Watchdog heartbeat.
+# Validates the effective heartbeat after Flux post-build substitution.
+assert_coroot_heartbeat_substitution() {
+  local rendered_path="$1"
+  local substituted_path
+  local heartbeat_url="https://hc-ping.com/example-heartbeat"
+  local heartbeat_host
+  local cronjob_matches
+  local policy_matches
+
+  substituted_path="${tmp_dir}/$(basename "${rendered_path}" .yaml)-heartbeat-substituted.yaml"
+  heartbeat_host="${heartbeat_url#https://}"
+  heartbeat_host="${heartbeat_host%%/*}"
+  alertmanager_heartbeat_url="${heartbeat_url}" flux envsubst \
+    <"${rendered_path}" >"${substituted_path}"
+
+  cronjob_matches="$(
+    yq eval-all -o=json '.' "${substituted_path}" |
+      jq -s --arg heartbeat_url "${heartbeat_url}" '[.[] |
+        select(.kind == "CronJob" and .metadata.namespace == "observability" and .metadata.name == "cluster-heartbeat") |
+        select(.spec.jobTemplate.spec.template.spec.containers[0].command[2] ==
+          ("curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"" + $heartbeat_url + "\" || true"))] | length'
+  )"
+  policy_matches="$(
+    yq eval-all -o=json '.' "${substituted_path}" |
+      jq -s --arg heartbeat_host "${heartbeat_host}" '[.[] |
+        select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-cluster-heartbeat") |
+        select(any(.spec.egress[]?.toFQDNs[]?; .matchName == $heartbeat_host)) |
+        select(any(.spec.egress[]?.toPorts[]?.rules.dns[]?; .matchName == $heartbeat_host))] | length'
+  )"
+
+  if [[ "${cronjob_matches}" != "1" || "${policy_matches}" != "1" ]]; then
+    echo "Flux-substituted heartbeat URL and exact policy host must agree in ${rendered_path}" >&2
+    return 1
+  fi
+}
+
+# Validates the expected Watchdog state in default and Coroot profiles.
 assert_watchdog_disabled() {
   local rendered_path="$1"
   local expected="$2"
@@ -262,6 +298,41 @@ assert_heartbeat_policy_exclusion() {
   done
 }
 
+# Verifies that default profiles retain the global deny and DNS selectors.
+assert_heartbeat_policy_exclusion_absent() {
+  local rendered_path="$1"
+  local rule_name
+  local matches
+
+  for rule_name in generate-default-deny generate-allow-dns; do
+    matches="$(
+      yq eval-all -o=json '.' "${rendered_path}" |
+        jq -s --arg rule_name "${rule_name}" '[.[] |
+          select(.kind == "ClusterPolicy" and .metadata.name == "add-default-deny") |
+          .spec.rules[] |
+          select(.name == $rule_name) |
+          select(((.exclude.any[0].resources.names // []) | index("observability")) == null) |
+          select(.generate.data.spec.endpointSelector == {})] | length'
+    )"
+    if [[ "${matches}" != "1" ]]; then
+      echo "default add-default-deny/${rule_name} must keep its global selector in ${rendered_path}" >&2
+      return 1
+    fi
+  done
+
+  matches="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.kind == "ClusterPolicy" and .metadata.name == "add-default-deny") |
+        .spec.rules[] |
+        select(.name == "generate-default-deny-observability" or .name == "generate-allow-dns-observability")] | length'
+  )"
+  if [[ "${matches}" != "0" ]]; then
+    echo "default add-default-deny must not contain Coroot heartbeat exceptions in ${rendered_path}" >&2
+    return 1
+  fi
+}
+
 assert_opencost_present() {
   local rendered_path="$1"
 
@@ -600,7 +671,7 @@ for rendered_path in "${local_infrastructure_default}" "${prod_infrastructure_de
   assert_security_exception_namespace_count "${rendered_path}" infrastructure-privileged observability 0
   assert_security_exception_namespace_count "${rendered_path}" controller-rbac observability 0
   assert_security_exception_namespace_count "${rendered_path}" service-account-tokens observability 0
-  assert_heartbeat_policy_exclusion "${rendered_path}"
+  assert_heartbeat_policy_exclusion_absent "${rendered_path}"
 done
 assert_opencost_reference_count "${local_apps_default}" 1
 assert_opencost_reference_count "${prod_apps_default}" 1
@@ -654,7 +725,8 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_opencost_absent "${rendered_path}"
   assert_coroot_sso_controller_contract "${rendered_path}"
   assert_coroot_heartbeat_contract "${rendered_path}"
-  assert_watchdog_disabled "${rendered_path}" true
+  assert_coroot_heartbeat_substitution "${rendered_path}"
+  assert_watchdog_disabled "${rendered_path}" false
 done
 
 for rendered_path in "${docker_infrastructure}" "${hetzner_infrastructure}"; do
@@ -820,12 +892,10 @@ for documented_boundary in \
   "retires the legacy Loki and Alloy log path" \
   "stages one hardened \`cluster-heartbeat\` CronJob" \
   "reuses the existing encrypted heartbeat URL" \
-  "configured heartbeat hostname's exact DNS query" \
-  "\`alertmanager_heartbeat_host\`" \
+  "permits only \`hc-ping.com:443\`" \
   "runs at platform-critical priority" \
-  "disables only Watchdog" \
-  "host is derived automatically" \
-  "Custom providers must accept HTTPS on port 443" \
+  "runs alongside Watchdog" \
+  "custom-provider heartbeats" \
   "does not add a new secret" \
   "keeps kube-prometheus-stack" \
   "Cost allocation is therefore unavailable" \
@@ -845,27 +915,21 @@ for documented_boundary in \
   fi
 done
 
-bootstrap_workflow="${repo_root}/.github/workflows/bootstrap.yaml"
-for heartbeat_host_boundary in \
-  "heartbeat_authority=\"\${ALERTMANAGER_HEARTBEAT_URL#*://}\"" \
-  "heartbeat_authority=\"\${heartbeat_authority%%/*}\"" \
-  "heartbeat_port=\"\${heartbeat_authority##*:}\"" \
-  "[[ \"\${heartbeat_port}\" != \"443\" ]]" \
-  "ALERTMANAGER_HEARTBEAT_HOST=\"\${heartbeat_authority%%:*}\"" \
-  '.stringData.alertmanager_heartbeat_host = strenv(ALERTMANAGER_HEARTBEAT_HOST)'; do
-  if ! grep -Fq "${heartbeat_host_boundary}" "${bootstrap_workflow}"; then
-    echo "bootstrap workflow does not derive the heartbeat policy host from the configured URL: ${heartbeat_host_boundary}" >&2
-    exit 1
-  fi
-done
+if grep -R -n 'alertmanager_heartbeat_host' \
+  "${repo_root}/.github/workflows/bootstrap.yaml" \
+  "${repo_root}/docs" \
+  "${repo_root}/k8s"; then
+  echo "Coroot heartbeat must not add an instance-owned host variable" >&2
+  exit 1
+fi
 
 alerting_guide="${repo_root}/docs/dr/alerting.md"
 for documented_heartbeat_boundary in \
   "The default profile keeps the \`Watchdog\` alert" \
   "The Coroot profile uses the dedicated \`cluster-heartbeat\` CronJob" \
-  "policy host is derived automatically" \
-  "alertmanager_heartbeat_host" \
-  "disables Watchdog" \
+  "only \`hc-ping.com:443\`" \
+  "keeps Watchdog" \
+  "custom-provider heartbeats" \
   "Flux reconciliation alerting still depends on kube-prometheus-stack"; do
   if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then
     echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2

From 52e6e9405f55190fc1300547aba00564aa0b90d5 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 23:47:47 +0200
Subject: [PATCH 14/26] fix(observability): keep validator self-contained

---
 scripts/validate-observability-option.sh | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 7ed6f33..22d0259 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -205,14 +205,25 @@ assert_coroot_heartbeat_substitution() {
   local substituted_path
   local heartbeat_url="https://hc-ping.com/example-heartbeat"
   local heartbeat_host
+  local heartbeat_token='$'
+  local rendered_line
+  local rendered_prefix
+  local rendered_suffix
   local cronjob_matches
   local policy_matches
 
   substituted_path="${tmp_dir}/$(basename "${rendered_path}" .yaml)-heartbeat-substituted.yaml"
+  heartbeat_token+='{alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}'
   heartbeat_host="${heartbeat_url#https://}"
   heartbeat_host="${heartbeat_host%%/*}"
-  alertmanager_heartbeat_url="${heartbeat_url}" flux envsubst \
-    <"${rendered_path}" >"${substituted_path}"
+  while IFS= read -r rendered_line || [[ -n "${rendered_line}" ]]; do
+    if [[ "${rendered_line}" == *"${heartbeat_token}"* ]]; then
+      rendered_prefix="${rendered_line%%"${heartbeat_token}"*}"
+      rendered_suffix="${rendered_line#*"${heartbeat_token}"}"
+      rendered_line="${rendered_prefix}${heartbeat_url}${rendered_suffix}"
+    fi
+    printf '%s\n' "${rendered_line}"
+  done <"${rendered_path}" >"${substituted_path}"
 
   cronjob_matches="$(
     yq eval-all -o=json '.' "${substituted_path}" |

From 6038a046b52f5d326ee313774b4c067a456a16e0 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Tue, 21 Jul 2026 23:54:08 +0200
Subject: [PATCH 15/26] fix(observability): isolate heartbeat input

---
 .../controllers/coroot/cron-job.yaml          |  6 ++-
 scripts/validate-observability-option.sh      | 43 ++++++++++++++-----
 2 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml b/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
index d32a9a4..0e7e58a 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
@@ -46,11 +46,13 @@ spec:
               command:
                 - /bin/sh
                 - -c
+              args:
                 - >-
                   curl -sf --max-time 10
                   --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused
-                  "${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}"
-                  || true
+                  "$1" || true
+                - cluster-heartbeat
+                - "${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}"
               securityContext:
                 allowPrivilegeEscalation: false
                 readOnlyRootFilesystem: true
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 22d0259..79842c8 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -147,8 +147,12 @@ assert_coroot_heartbeat_contract() {
         select($container.image == "docker.io/curlimages/curl:8.21.0@sha256:7c12af72ceb38b7432ab85e1a265cff6ae58e06f95539d539b654f2cfa64bb13") |
         select($container.command == [
           "/bin/sh",
-          "-c",
-          "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}\" || true"
+          "-c"
+        ]) |
+        select($container.args == [
+          "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"$1\" || true",
+          "cluster-heartbeat",
+          "${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}"
         ]) |
         select($container.securityContext.allowPrivilegeEscalation == false) |
         select($container.securityContext.readOnlyRootFilesystem == true) |
@@ -229,8 +233,13 @@ assert_coroot_heartbeat_substitution() {
     yq eval-all -o=json '.' "${substituted_path}" |
       jq -s --arg heartbeat_url "${heartbeat_url}" '[.[] |
         select(.kind == "CronJob" and .metadata.namespace == "observability" and .metadata.name == "cluster-heartbeat") |
-        select(.spec.jobTemplate.spec.template.spec.containers[0].command[2] ==
-          ("curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"" + $heartbeat_url + "\" || true"))] | length'
+        .spec.jobTemplate.spec.template.spec.containers[0] as $container |
+        select($container.command == ["/bin/sh", "-c"]) |
+        select($container.args == [
+          "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"$1\" || true",
+          "cluster-heartbeat",
+          $heartbeat_url
+        ])] | length'
   )"
   policy_matches="$(
     yq eval-all -o=json '.' "${substituted_path}" |
@@ -271,6 +280,7 @@ assert_heartbeat_policy_exclusion() {
   local rendered_path="$1"
   local rule_name
   local scoped_rule_name
+  local expected_spec
   local matches
 
   for rule_name in generate-default-deny generate-allow-dns; do
@@ -289,21 +299,34 @@ assert_heartbeat_policy_exclusion() {
     fi
 
     scoped_rule_name="${rule_name}-observability"
+    if [[ "${rule_name}" == "generate-default-deny" ]]; then
+      expected_spec='{
+        "endpointSelector":{"matchExpressions":[{"key":"platform-heartbeat","operator":"DoesNotExist"}]},
+        "enableDefaultDeny":{"ingress":true,"egress":true},
+        "ingressDeny":[{}],
+        "egressDeny":[{}]
+      }'
+    else
+      expected_spec='{
+        "endpointSelector":{"matchExpressions":[{"key":"platform-heartbeat","operator":"DoesNotExist"}]},
+        "egress":[{
+          "toEndpoints":[{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}],
+          "toPorts":[{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}]}]
+        }]
+      }'
+    fi
     matches="$(
       yq eval-all -o=json '.' "${rendered_path}" |
-        jq -s --arg rule_name "${scoped_rule_name}" '[.[] |
+        jq -s --arg rule_name "${scoped_rule_name}" --argjson expected_spec "${expected_spec}" '[.[] |
           select(.kind == "ClusterPolicy" and .metadata.name == "add-default-deny") |
           .spec.rules[] |
           select(.name == $rule_name) |
           select(.match.any[0].resources.kinds == ["Namespace"]) |
           select(.match.any[0].resources.names == ["observability"]) |
-          select(.generate.data.spec.endpointSelector.matchExpressions == [{
-            "key":"platform-heartbeat",
-            "operator":"DoesNotExist"
-          }])] | length'
+          select(.generate.data.spec == $expected_spec)] | length'
     )"
     if [[ "${matches}" != "1" ]]; then
-      echo "add-default-deny/${scoped_rule_name} must scope the heartbeat exclusion to observability in ${rendered_path}" >&2
+      echo "add-default-deny/${scoped_rule_name} must retain its complete scoped policy body in ${rendered_path}" >&2
       return 1
     fi
   done

From 4b2c52e025d3b2079764ffb879d1c227b7fcfdac Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 00:11:13 +0200
Subject: [PATCH 16/26] fix(observability): close heartbeat policy gaps

---
 .../coroot/cilium-network-policy.yaml         |  4 +-
 .../isolate-heartbeat/kustomization.yaml      |  8 ++-
 scripts/validate-observability-option.sh      | 58 +++++++++++--------
 3 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
index ad82850..d8e6b5c 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml
@@ -7,7 +7,9 @@ spec:
   endpointSelector:
     matchExpressions:
       - key: platform-heartbeat
-        operator: DoesNotExist
+        operator: NotIn
+        values:
+          - "true"
   ingress:
     # Coroot, its bundled stores, its agents, and the audit forwarder exchange
     # telemetry inside this namespace.
diff --git a/k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml b/k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml
index b42c58c..d3302e0 100644
--- a/k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml
+++ b/k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml
@@ -51,7 +51,9 @@ patches:
                 endpointSelector:
                   matchExpressions:
                     - key: platform-heartbeat
-                      operator: DoesNotExist
+                      operator: NotIn
+                      values:
+                        - "true"
                 enableDefaultDeny:
                   ingress: true
                   egress: true
@@ -82,7 +84,9 @@ patches:
                 endpointSelector:
                   matchExpressions:
                     - key: platform-heartbeat
-                      operator: DoesNotExist
+                      operator: NotIn
+                      values:
+                        - "true"
                 egress:
                   - toEndpoints:
                       - matchLabels:
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 79842c8..7a6086e 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -167,22 +167,30 @@ assert_coroot_heartbeat_contract() {
     yq eval-all -o=json '.' "${rendered_path}" |
       jq -s '[.[] |
         select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-cluster-heartbeat") |
-        select(.spec.endpointSelector.matchLabels == {"platform-heartbeat":"true"}) |
-        select(.spec.enableDefaultDeny == {"ingress":true,"egress":true}) |
-        select(.spec.ingressDeny == [{}]) |
-        select(.spec.egress | length == 2) |
-        select(any(.spec.egress[];
-          .toFQDNs == [{"matchName":"hc-ping.com"}] and
-          .toPorts == [{"ports":[{"port":"443","protocol":"TCP"}]}])) |
-        select(any(.spec.egress[];
-          .toEndpoints == [{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}] and
-          .toPorts == [{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}],"rules":{"dns":[{"matchName":"hc-ping.com"}]}}]))] | length'
+        select(.spec == {
+          "endpointSelector":{"matchLabels":{"platform-heartbeat":"true"}},
+          "enableDefaultDeny":{"ingress":true,"egress":true},
+          "ingressDeny":[{}],
+          "egress":[
+            {
+              "toFQDNs":[{"matchName":"hc-ping.com"}],
+              "toPorts":[{"ports":[{"port":"443","protocol":"TCP"}]}]
+            },
+            {
+              "toEndpoints":[{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}],
+              "toPorts":[{
+                "ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}],
+                "rules":{"dns":[{"matchName":"hc-ping.com"}]}
+              }]
+            }
+          ]
+        })] | length'
   )"
   coroot_selector_contract="$(
     yq eval-all -o=json '.' "${rendered_path}" |
       jq -s '[.[] |
         select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-coroot") |
-        select(.spec.endpointSelector.matchExpressions == [{"key":"platform-heartbeat","operator":"DoesNotExist"}]) |
+        select(.spec.endpointSelector.matchExpressions == [{"key":"platform-heartbeat","operator":"NotIn","values":["true"]}]) |
         select(all(.spec.egress[]?.toFQDNs[]?; .matchName != "hc-ping.com"))] | length'
   )"
   broad_external_egress="$(
@@ -255,21 +263,21 @@ assert_coroot_heartbeat_substitution() {
   fi
 }
 
-# Validates the expected Watchdog state in default and Coroot profiles.
-assert_watchdog_disabled() {
+# Validates that kube-prometheus-stack still produces the Watchdog rule.
+assert_watchdog_rendered() {
   local rendered_path="$1"
-  local expected="$2"
-  local actual
+  local matches
 
-  actual="$(
+  matches="$(
     yq eval-all -o=json '.' "${rendered_path}" |
-      jq -s -r '[.[] |
+      jq -s '[.[] |
         select(.kind == "HelmRelease" and .metadata.namespace == "monitoring" and .metadata.name == "kube-prometheus-stack") |
-        (.spec.values.defaultRules.disabled.Watchdog // false)] |
-        if length == 1 then .[0] else "invalid-count:\(length)" end'
+        select(.spec.values.defaultRules.create == true) |
+        select((.spec.values.defaultRules.rules.general // true) == true) |
+        select((.spec.values.defaultRules.disabled.Watchdog // false) == false)] | length'
   )"
-  if [[ "${actual}" != "${expected}" ]]; then
-    echo "expected kube-prometheus-stack Watchdog disabled=${expected} in ${rendered_path}, found ${actual}" >&2
+  if [[ "${matches}" != "1" ]]; then
+    echo "kube-prometheus-stack must render its enabled general/Watchdog rule in ${rendered_path}" >&2
     return 1
   fi
 }
@@ -301,14 +309,14 @@ assert_heartbeat_policy_exclusion() {
     scoped_rule_name="${rule_name}-observability"
     if [[ "${rule_name}" == "generate-default-deny" ]]; then
       expected_spec='{
-        "endpointSelector":{"matchExpressions":[{"key":"platform-heartbeat","operator":"DoesNotExist"}]},
+        "endpointSelector":{"matchExpressions":[{"key":"platform-heartbeat","operator":"NotIn","values":["true"]}]},
         "enableDefaultDeny":{"ingress":true,"egress":true},
         "ingressDeny":[{}],
         "egressDeny":[{}]
       }'
     else
       expected_spec='{
-        "endpointSelector":{"matchExpressions":[{"key":"platform-heartbeat","operator":"DoesNotExist"}]},
+        "endpointSelector":{"matchExpressions":[{"key":"platform-heartbeat","operator":"NotIn","values":["true"]}]},
         "egress":[{
           "toEndpoints":[{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}],
           "toPorts":[{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}]}]
@@ -695,7 +703,7 @@ done
 # Only the explicit Coroot profiles retire it.
 for rendered_path in "${local_controllers_default}" "${prod_controllers_default}"; do
   assert_opencost_present "${rendered_path}"
-  assert_watchdog_disabled "${rendered_path}" false
+  assert_watchdog_rendered "${rendered_path}"
 done
 for rendered_path in "${local_infrastructure_default}" "${prod_infrastructure_default}"; do
   assert_opencost_resources_absent "${rendered_path}"
@@ -760,7 +768,7 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_coroot_sso_controller_contract "${rendered_path}"
   assert_coroot_heartbeat_contract "${rendered_path}"
   assert_coroot_heartbeat_substitution "${rendered_path}"
-  assert_watchdog_disabled "${rendered_path}" false
+  assert_watchdog_rendered "${rendered_path}"
 done
 
 for rendered_path in "${docker_infrastructure}" "${hetzner_infrastructure}"; do

From 04fa044ec720c6b990946e46a71a2629c5c6db56 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 01:12:29 +0200
Subject: [PATCH 17/26] docs(observability): document render count helpers

---
 scripts/validate-observability-option.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 7a6086e..25c19ce 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -16,6 +16,7 @@ render() {
   fi
 }
 
+# Counts one exact kind/namespace/name identity in a multi-document render.
 resource_count() {
   local rendered_path="$1"
   local kind="$2"
@@ -61,6 +62,7 @@ loki_service_reference_count() {
     jq -s '[.[] | select(type == "object" and (tojson | test("loki\\.monitoring\\.svc")))] | length'
 }
 
+# Fails closed when an exact rendered identity is missing or duplicated.
 assert_resource_count() {
   local rendered_path="$1"
   local kind="$2"

From ebd7c85e02cf6fb0e25c7de8ed37dfa326f668bd Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 03:14:07 +0200
Subject: [PATCH 18/26] docs(observability): explain render helper

---
 scripts/validate-observability-option.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 25c19ce..04cdeeb 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -6,6 +6,8 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 tmp_dir="$(mktemp -d)"
 trap 'rm -rf "${tmp_dir}"' EXIT
 
+# Renders a repository-relative profile so every assertion evaluates the effective
+# Kustomize payload independently of the caller's working directory.
 render() {
   local source_path="$1"
   local output_path="$2"

From 2ad7cbd143061f695f17fce70bd87c05af7df78a Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 03:20:06 +0200
Subject: [PATCH 19/26] docs(observability): align helper documentation

---
 scripts/validate-observability-option.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 04cdeeb..acdc557 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -6,8 +6,7 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 tmp_dir="$(mktemp -d)"
 trap 'rm -rf "${tmp_dir}"' EXIT
 
-# Renders a repository-relative profile so every assertion evaluates the effective
-# Kustomize payload independently of the caller's working directory.
+# Renders a repository-relative profile independently of the caller's working directory.
 render() {
   local source_path="$1"
   local output_path="$2"

From e7a11dde53bc7e9c2e144a186ebeb9f603099949 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 03:47:57 +0200
Subject: [PATCH 20/26] fix(observability): separate Coroot heartbeat signal

---
 docs/TEMPLATING.md                            | 12 +++---
 docs/dr/alerting.md                           | 17 ++++----
 .../disable-watchdog/kustomization.yaml       | 31 ++++++++++++++
 .../kustomization.yaml                        |  1 +
 .../kustomization.yaml                        |  1 +
 scripts/validate-observability-option.sh      | 41 ++++++++++++++-----
 6 files changed, 80 insertions(+), 23 deletions(-)
 create mode 100644 k8s/components/observability-coroot/disable-watchdog/kustomization.yaml

diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index 34bfdb2..ab08822 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -95,11 +95,13 @@ plus that hostname's exact DNS query. The Coroot infrastructure
 overlay alone narrows the generated namespace policies around the CronJob's
 purpose-specific label, so default profiles retain their global selectors. The
 invalid URL fallback keeps local and unconfigured instances inert. During this
-staging slice the CronJob runs alongside Watchdog: kube-prometheus-stack keeps the
-existing dead-man path so already-synced custom-provider heartbeats cannot be lost
-before a later, independently validated cutover. Kube-prometheus-stack keeps owning its alert rules,
-including Flux reconciliation alerts, until that
-remaining migration is delivered separately.
+staging slice the Coroot profile disables Watchdog. The CronJob exclusively owns
+the external heartbeat. Two independent senders must not reset the same monitor:
+the CronJob would otherwise hide a Prometheus-to-Alertmanager outage by continuing
+to ping while that alerting path was down. Default profiles keep Watchdog unchanged.
+Kube-prometheus-stack keeps owning its remaining alert
+rules, including Flux reconciliation alerts, until that migration is delivered
+separately.
 
 Open the Coroot UI at `https://observability.<your-domain>`. Community Edition
 does not provide native OIDC, so this profile sends every UI request through the
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index ade6d47..df73633 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -57,14 +57,15 @@ tight cadence (`repeat_interval: 50s`). If the cluster — or the Prometheus →
 Alertmanager pipeline — dies, the monitor stops receiving pings and notifies
 Slack out-of-band.
 
-The Coroot profile uses the dedicated `cluster-heartbeat` CronJob alongside that
-path. It pings the same existing URL every five minutes, independently of Coroot,
-and its network policy allows only `hc-ping.com:443` plus that hostname's exact
-DNS query. The Coroot overlay alone narrows the namespace deny and DNS policies
-around this purpose-labelled workload; default profiles keep their global
-selectors. The invalid URL fallback keeps local and unconfigured instances quiet.
-Coroot keeps Watchdog during this staging slice so custom-provider heartbeats remain
-protected before a later, independently validated cutover.
+The Coroot profile gives the existing URL exclusively to the dedicated
+`cluster-heartbeat` CronJob, which pings every five minutes independently of
+Coroot. That profile disables Watchdog: if both senders reset the same monitor,
+the CronJob can keep it green while the Prometheus-to-Alertmanager pipeline is
+down. The CronJob's network policy allows only `hc-ping.com:443` plus that
+hostname's exact DNS query. The Coroot overlay alone narrows the namespace deny
+and DNS policies around this purpose-labelled workload; default profiles keep
+their global selectors. Their Watchdog path stays unchanged. The invalid URL
+fallback keeps local and unconfigured instances quiet.
 Flux reconciliation alerting still depends on kube-prometheus-stack, so that stack
 remains transitional until a later slice replaces those rules before removing it.
 
diff --git a/k8s/components/observability-coroot/disable-watchdog/kustomization.yaml b/k8s/components/observability-coroot/disable-watchdog/kustomization.yaml
new file mode 100644
index 0000000..c6d9d32
--- /dev/null
+++ b/k8s/components/observability-coroot/disable-watchdog/kustomization.yaml
@@ -0,0 +1,31 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+# The dedicated CronJob owns the Coroot profile's external heartbeat. Disable
+# Watchdog there so two independent senders cannot reset the same monitor and
+# hide a Prometheus-to-Alertmanager outage. Default profiles keep Watchdog.
+patches:
+  - target:
+      group: helm.toolkit.fluxcd.io
+      version: v2
+      kind: HelmRelease
+      name: kube-prometheus-stack
+      namespace: monitoring
+    patch: |
+      - op: test
+        path: /spec/values/defaultRules/create
+        value: true
+      - op: test
+        path: /spec/values/defaultRules/disabled/KubeCPUOvercommit
+        value: true
+      - op: test
+        path: /spec/values/alertmanager/config/route/routes/0/receiver
+        value: heartbeat
+      - op: test
+        path: /spec/values/alertmanager/config/receivers/1/name
+        value: heartbeat
+      - op: test
+        path: /spec/values/alertmanager/config/receivers/1/webhook_configs/0/url
+        value: ${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}
+      - op: add
+        path: /spec/values/defaultRules/disabled/Watchdog
+        value: true
diff --git a/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml b/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
index 16a6418..2b2b312 100644
--- a/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
+++ b/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
@@ -5,6 +5,7 @@ resources:
   - ../infrastructure/controllers/
   - ../../../bases/infrastructure/controllers/coroot/
 components:
+  - ../../../components/observability-coroot/disable-watchdog/
   - ../../../components/observability-coroot/remove-controllers-loki-alloy/
   - ../../../components/observability-coroot/remove-controller-opencost/
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
index 3688282..91ba52b 100644
--- a/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
+++ b/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
@@ -5,6 +5,7 @@ resources:
   - ../infrastructure/controllers/
   - ../../../bases/infrastructure/controllers/coroot/
 components:
+  - ../../../components/observability-coroot/disable-watchdog/
   - ../../../components/observability-coroot/remove-controllers-loki-alloy/
   - ../../../components/observability-coroot/remove-controller-opencost/
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index acdc557..801cca6 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -266,8 +266,8 @@ assert_coroot_heartbeat_substitution() {
   fi
 }
 
-# Validates that kube-prometheus-stack still produces the Watchdog rule.
-assert_watchdog_rendered() {
+# Validates that the default profile still produces the Watchdog rule.
+assert_watchdog_enabled() {
   local rendered_path="$1"
   local matches
 
@@ -285,6 +285,26 @@ assert_watchdog_rendered() {
   fi
 }
 
+# Validates that a Coroot profile assigns the external heartbeat exclusively to
+# the dedicated CronJob instead of letting Watchdog reset the same monitor.
+assert_watchdog_disabled() {
+  local rendered_path="$1"
+  local matches
+
+  matches="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.kind == "HelmRelease" and .metadata.namespace == "monitoring" and .metadata.name == "kube-prometheus-stack") |
+        select(.spec.values.defaultRules.create == true) |
+        select((.spec.values.defaultRules.rules.general // true) == true) |
+        select(.spec.values.defaultRules.disabled.Watchdog == true)] | length'
+  )"
+  if [[ "${matches}" != "1" ]]; then
+    echo "Coroot profile must disable Watchdog while cluster-heartbeat owns the external monitor in ${rendered_path}" >&2
+    return 1
+  fi
+}
+
 # Validates that only the observability namespace's generated policies defer
 # the dedicated heartbeat label to its narrower workload policy.
 assert_heartbeat_policy_exclusion() {
@@ -706,7 +726,7 @@ done
 # Only the explicit Coroot profiles retire it.
 for rendered_path in "${local_controllers_default}" "${prod_controllers_default}"; do
   assert_opencost_present "${rendered_path}"
-  assert_watchdog_rendered "${rendered_path}"
+  assert_watchdog_enabled "${rendered_path}"
 done
 for rendered_path in "${local_infrastructure_default}" "${prod_infrastructure_default}"; do
   assert_opencost_resources_absent "${rendered_path}"
@@ -771,7 +791,7 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_coroot_sso_controller_contract "${rendered_path}"
   assert_coroot_heartbeat_contract "${rendered_path}"
   assert_coroot_heartbeat_substitution "${rendered_path}"
-  assert_watchdog_rendered "${rendered_path}"
+  assert_watchdog_disabled "${rendered_path}"
 done
 
 for rendered_path in "${docker_infrastructure}" "${hetzner_infrastructure}"; do
@@ -939,8 +959,9 @@ for documented_boundary in \
   "reuses the existing encrypted heartbeat URL" \
   "permits only \`hc-ping.com:443\`" \
   "runs at platform-critical priority" \
-  "runs alongside Watchdog" \
-  "custom-provider heartbeats" \
+  "The CronJob exclusively owns" \
+  "disables Watchdog" \
+  "Default profiles keep Watchdog unchanged" \
   "does not add a new secret" \
   "keeps kube-prometheus-stack" \
   "Cost allocation is therefore unavailable" \
@@ -949,7 +970,7 @@ for documented_boundary in \
   "permits only \`hooks.slack.com:443\`" \
   "Per-alert notifications remain visible only in the Coroot UI" \
   "Local / Docker Coroot stays notification-free" \
-  "Kube-prometheus-stack keeps owning its alert rules" \
+  "Kube-prometheus-stack keeps owning its remaining alert" \
   "https://observability.<your-domain>" \
   "Dex-backed oauth2-proxy" \
   "no direct Gateway route to the Coroot service" \
@@ -971,10 +992,10 @@ fi
 alerting_guide="${repo_root}/docs/dr/alerting.md"
 for documented_heartbeat_boundary in \
   "The default profile keeps the \`Watchdog\` alert" \
-  "The Coroot profile uses the dedicated \`cluster-heartbeat\` CronJob" \
+  "The Coroot profile gives the existing URL exclusively" \
   "only \`hc-ping.com:443\`" \
-  "keeps Watchdog" \
-  "custom-provider heartbeats" \
+  "disables Watchdog" \
+  "Watchdog path stays unchanged" \
   "Flux reconciliation alerting still depends on kube-prometheus-stack"; do
   if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then
     echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2

From f010e85729c28c5e752098ba46b488a676c2a26a Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 04:10:33 +0200
Subject: [PATCH 21/26] fix(observability): separate heartbeat monitors

---
 .github/workflows/bootstrap.yaml              | 14 ++++
 docs/BOOTSTRAP.md                             |  3 +-
 docs/TEMPLATING.md                            | 18 ++---
 docs/dr/alerting.md                           | 56 +++++++++------
 .../controllers/coroot/cron-job.yaml          |  2 +-
 .../disable-watchdog/kustomization.yaml       | 31 --------
 .../kustomization.yaml                        |  1 -
 .../kustomization.yaml                        |  1 -
 scripts/validate-observability-option.sh      | 72 ++++++++++---------
 9 files changed, 97 insertions(+), 101 deletions(-)
 delete mode 100644 k8s/components/observability-coroot/disable-watchdog/kustomization.yaml

diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml
index 094a1e2..612e61b 100644
--- a/.github/workflows/bootstrap.yaml
+++ b/.github/workflows/bootstrap.yaml
@@ -185,6 +185,7 @@ jobs:
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           ALERTMANAGER_WEBHOOK_URL: ${{ secrets.ALERTMANAGER_WEBHOOK_URL }}
           ALERTMANAGER_HEARTBEAT_URL: ${{ secrets.ALERTMANAGER_HEARTBEAT_URL }}
+          CLUSTER_HEARTBEAT_URL: ${{ secrets.CLUSTER_HEARTBEAT_URL }}
           GITHUB_APP_CLIENT_SECRET: ${{ secrets.GITHUB_APP_CLIENT_SECRET }}
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
         run: |
@@ -194,6 +195,18 @@ jobs:
           export FLUX_WEB_CLIENT_SECRET="$(openssl rand -hex 32)"
           export OAUTH2_PROXY_COOKIE_SECRET="$(openssl rand -base64 32)"
 
+          # The dedicated Coroot heartbeat has an exact hc-ping.com egress
+          # policy. Reject a configured URL that the cluster cannot reach rather
+          # than encrypting a silently dead monitor. Empty stays valid so default
+          # profiles and operators who have not opted into Coroot remain inert.
+          case "${CLUSTER_HEARTBEAT_URL}" in
+            ""|https://hc-ping.com/*) ;;
+            *)
+              echo "::error::CLUSTER_HEARTBEAT_URL must use https://hc-ping.com/ or be unset"
+              exit 1
+              ;;
+          esac
+
           fill_base() { # variables-base-secret
             yq -i '
               .stringData.cloudflare_api_token = strenv(CLOUDFLARE_API_TOKEN) |
@@ -207,6 +220,7 @@ jobs:
           yq -i '
             .stringData.alertmanager_webhook_url = strenv(ALERTMANAGER_WEBHOOK_URL) |
             .stringData.alertmanager_heartbeat_url = strenv(ALERTMANAGER_HEARTBEAT_URL) |
+            .stringData.cluster_heartbeat_url = strenv(CLUSTER_HEARTBEAT_URL) |
             .stringData.dex_client_secret = strenv(DEX_CLIENT_SECRET) |
             .stringData.flux_web_client_secret = strenv(FLUX_WEB_CLIENT_SECRET) |
             .stringData.oauth2_proxy_cookie_secret = strenv(OAUTH2_PROXY_COOKIE_SECRET) |
diff --git a/docs/BOOTSTRAP.md b/docs/BOOTSTRAP.md
index f868851..acf014a 100644
--- a/docs/BOOTSTRAP.md
+++ b/docs/BOOTSTRAP.md
@@ -111,7 +111,8 @@ Set these in your **new instance repository** under *Settings → Secrets and va
 | `R2_ACCESS_KEY_ID` | R2 (S3) access key ID for backups | Cloudflare R2 → **Manage R2 API Tokens**. |
 | `R2_SECRET_ACCESS_KEY` | R2 (S3) secret access key for backups | Same place as the access key ID. |
 | `ALERTMANAGER_WEBHOOK_URL` | Incoming webhook for alert notifications (e.g. Slack) | Your chat platform’s incoming-webhook setup. |
-| `ALERTMANAGER_HEARTBEAT_URL` | Ping URL for the off-cluster dead-man’s-switch | healthchecks.io (or similar) check ping URL. |
+| `ALERTMANAGER_HEARTBEAT_URL` | Ping URL that proves the Prometheus → Alertmanager pipeline is alive | healthchecks.io (or similar) check ping URL. |
+| `CLUSTER_HEARTBEAT_URL` | Optional Coroot-profile cluster-liveness ping URL | A **separate** `https://hc-ping.com/...` check; never reuse the Alertmanager check. |
 | `GITHUB_APP_CLIENT_SECRET` | **Client secret** of the *cluster SSO* GitHub App (Dex’s GitHub connector) | The SSO App’s settings page (generate a client secret). |
 | `APP_PRIVATE_KEY` | Private key (`.pem`) of the **bootstrap** GitHub App | Generated when you created the bootstrap App. |
 
diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index ab08822..5bf0dcb 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -89,16 +89,18 @@ policy permits only `hooks.slack.com:443`, so keep this value a Slack incoming-w
 Per-alert notifications remain visible only in the Coroot UI; this avoids sending
 every noisy alert to the shared webhook. Local / Docker Coroot stays notification-free.
 The profile also stages one hardened `cluster-heartbeat` CronJob for the cluster
-dead-man signal. It reuses the existing encrypted heartbeat URL and does not add a new secret.
-It runs at platform-critical priority, and its dedicated policy permits only `hc-ping.com:443`
-plus that hostname's exact DNS query. The Coroot infrastructure
+dead-man signal. It uses a separate optional `CLUSTER_HEARTBEAT_URL` secret so
+the existing Alertmanager Watchdog check keeps proving the Prometheus-to-Alertmanager
+pipeline. Never point both inputs at the same monitor. The CronJob runs at
+platform-critical priority, and its dedicated policy permits only `hc-ping.com:443`
+plus that hostname's exact DNS query; bootstrap rejects any other configured host.
+The Coroot infrastructure
 overlay alone narrows the generated namespace policies around the CronJob's
 purpose-specific label, so default profiles retain their global selectors. The
-invalid URL fallback keeps local and unconfigured instances inert. During this
-staging slice the Coroot profile disables Watchdog. The CronJob exclusively owns
-the external heartbeat. Two independent senders must not reset the same monitor:
-the CronJob would otherwise hide a Prometheus-to-Alertmanager outage by continuing
-to ping while that alerting path was down. Default profiles keep Watchdog unchanged.
+invalid URL fallback keeps local and unconfigured instances inert. Both default
+and Coroot profiles keep Watchdog unchanged on the distinct Alertmanager pipeline
+monitor. This also preserves a dead-man signal if the later Coroot infrastructure
+layer has not yet established the CronJob's generated-policy isolation.
 Kube-prometheus-stack keeps owning its remaining alert
 rules, including Flux reconciliation alerts, until that migration is delivered
 separately.
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index df73633..ca54d4c 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -6,9 +6,9 @@ shipper) and **OpenCost** (cost), running per cluster. Self-hosted, no
 SaaS metrics tier. The stack is production-hardened along four axes:
 
 1. **Alerts go to Slack** via Alertmanager's native `slack_configs`.
-2. **A dead-man's-switch** (the always-firing `Watchdog` alert) is pushed
-   to an external heartbeat monitor — the one failure mode in-cluster
-   alerting can never cover (the whole cluster being down).
+2. **Two independent dead-man checks** separate the Prometheus → Alertmanager
+   pipeline from direct cluster liveness, so one sender cannot hide the other
+   path's failure.
 3. **State is persistent** — Prometheus, Alertmanager and Loki use Hetzner
    Cloud block volumes in prod, and Velero ships the `monitoring`
    namespace to R2 daily (24 h RPO).
@@ -57,28 +57,34 @@ tight cadence (`repeat_interval: 50s`). If the cluster — or the Prometheus →
 Alertmanager pipeline — dies, the monitor stops receiving pings and notifies
 Slack out-of-band.
 
-The Coroot profile gives the existing URL exclusively to the dedicated
-`cluster-heartbeat` CronJob, which pings every five minutes independently of
-Coroot. That profile disables Watchdog: if both senders reset the same monitor,
-the CronJob can keep it green while the Prometheus-to-Alertmanager pipeline is
-down. The CronJob's network policy allows only `hc-ping.com:443` plus that
-hostname's exact DNS query. The Coroot overlay alone narrows the namespace deny
-and DNS policies around this purpose-labelled workload; default profiles keep
-their global selectors. Their Watchdog path stays unchanged. The invalid URL
-fallback keeps local and unconfigured instances quiet.
+The Coroot profile adds a dedicated `cluster-heartbeat` CronJob, which pings a
+**second external check** every five minutes independently of Coroot. It never
+reuses `alertmanager_heartbeat_url`: doing so would let the CronJob keep that
+monitor green while the Prometheus-to-Alertmanager pipeline was down.
+Watchdog stays enabled on its existing check in both default and Coroot profiles.
+The CronJob's separate `cluster_heartbeat_url` accepts only `https://hc-ping.com/...`;
+its network policy allows only `hc-ping.com:443` plus that hostname's exact DNS
+query, and bootstrap rejects another configured host. The Coroot overlay alone
+narrows the namespace deny and DNS policies around this purpose-labelled workload;
+default profiles keep their global selectors. Keeping Watchdog active also preserves
+the original dead-man path if the later infrastructure reconciliation has not yet
+established that isolation. An invalid fallback keeps an unconfigured CronJob quiet.
 Flux reconciliation alerting still depends on kube-prometheus-stack, so that stack
 remains transitional until a later slice replaces those rules before removing it.
 
 Recommended monitor: [healthchecks.io](https://healthchecks.io) (free,
-open-source, native Slack integration). Create a check with a ~5 min
-period and ~10 min grace, connect it to Slack, and put its ping URL in
-`alertmanager_heartbeat_url` (below). A self-hosted alternative is a
+open-source, native Slack integration). Keep two distinct checks: the existing
+tight-cadence Watchdog URL in `alertmanager_heartbeat_url`, and a ~5 min period /
+~10 min grace direct-cluster check in `cluster_heartbeat_url`. Connect both to
+Slack, but never give them the same ping URL. A self-hosted alternative for the
+Alertmanager check is supported; the CronJob deliberately accepts only
+`hc-ping.com` because its egress policy is fail closed. Another alternative is a
 scheduled GitHub Actions workflow that probes the public Gateway and posts
 to Slack — fully under your control, no third-party monitor.
 
-The heartbeat URL is injected by Flux substitution
-(`${alertmanager_heartbeat_url}`); unset, it defaults to an invalid URL,
-so local/CI simply never heartbeat — harmless.
+Both URLs are injected by Flux substitution. An unset `cluster_heartbeat_url`
+defaults to an invalid URL, so local/CI and Coroot profiles without the optional
+second check stay inert while Watchdog continues unchanged.
 
 ## Off-cluster metric/log backup
 
@@ -141,9 +147,13 @@ which is a Flux `substituteFrom` source.
 sops --set '["stringData"]["alertmanager_webhook_url"] "https://hooks.slack.com/services/XXX/YYY/ZZZ"' \
   k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
 
-# 2. External heartbeat-monitor ping URL (e.g. healthchecks.io).
+# 2. Prometheus → Alertmanager pipeline check (healthchecks.io or similar).
 sops --set '["stringData"]["alertmanager_heartbeat_url"] "https://hc-ping.com/<uuid>"' \
   k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
+
+# 3. Separate direct-cluster check for the Coroot profile (hc-ping.com only).
+sops --set '["stringData"]["cluster_heartbeat_url"] "https://hc-ping.com/<different-uuid>"' \
+  k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
 ```
 
 Slack side: create an incoming webhook for your alerts channel (e.g.
@@ -151,10 +161,10 @@ Slack side: create an incoming webhook for your alerts channel (e.g.
 webhook posts to the channel it was created for. healthchecks.io side: create
 the check, connect its Slack integration, copy the ping URL.
 
-| Env   | `alertmanager_webhook_url`        | `alertmanager_heartbeat_url`       |
-| ----- | --------------------------------- | ---------------------------------- |
-| local | invalid URL (alerts stay local)   | unset → invalid (no heartbeat)     |
-| prod  | Slack alerts-channel webhook      | healthchecks.io ping URL           |
+| Env   | `alertmanager_webhook_url`      | `alertmanager_heartbeat_url`       | `cluster_heartbeat_url`                 |
+| ----- | ------------------------------- | ---------------------------------- | --------------------------------------- |
+| local | invalid URL (alerts stay local) | unset → invalid                    | unset → invalid                         |
+| prod  | Slack alerts-channel webhook    | pipeline-monitor ping URL          | distinct Coroot cluster-monitor ping URL |
 
 ## On-call: silence and inspect
 
diff --git a/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml b/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
index 0e7e58a..eb4e584 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
@@ -52,7 +52,7 @@ spec:
                   --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused
                   "$1" || true
                 - cluster-heartbeat
-                - "${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}"
+                - "${cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}"
               securityContext:
                 allowPrivilegeEscalation: false
                 readOnlyRootFilesystem: true
diff --git a/k8s/components/observability-coroot/disable-watchdog/kustomization.yaml b/k8s/components/observability-coroot/disable-watchdog/kustomization.yaml
deleted file mode 100644
index c6d9d32..0000000
--- a/k8s/components/observability-coroot/disable-watchdog/kustomization.yaml
+++ /dev/null
@@ -1,31 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1alpha1
-kind: Component
-# The dedicated CronJob owns the Coroot profile's external heartbeat. Disable
-# Watchdog there so two independent senders cannot reset the same monitor and
-# hide a Prometheus-to-Alertmanager outage. Default profiles keep Watchdog.
-patches:
-  - target:
-      group: helm.toolkit.fluxcd.io
-      version: v2
-      kind: HelmRelease
-      name: kube-prometheus-stack
-      namespace: monitoring
-    patch: |
-      - op: test
-        path: /spec/values/defaultRules/create
-        value: true
-      - op: test
-        path: /spec/values/defaultRules/disabled/KubeCPUOvercommit
-        value: true
-      - op: test
-        path: /spec/values/alertmanager/config/route/routes/0/receiver
-        value: heartbeat
-      - op: test
-        path: /spec/values/alertmanager/config/receivers/1/name
-        value: heartbeat
-      - op: test
-        path: /spec/values/alertmanager/config/receivers/1/webhook_configs/0/url
-        value: ${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}
-      - op: add
-        path: /spec/values/defaultRules/disabled/Watchdog
-        value: true
diff --git a/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml b/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
index 2b2b312..16a6418 100644
--- a/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
+++ b/k8s/providers/docker/infrastructure-controllers-coroot/kustomization.yaml
@@ -5,7 +5,6 @@ resources:
   - ../infrastructure/controllers/
   - ../../../bases/infrastructure/controllers/coroot/
 components:
-  - ../../../components/observability-coroot/disable-watchdog/
   - ../../../components/observability-coroot/remove-controllers-loki-alloy/
   - ../../../components/observability-coroot/remove-controller-opencost/
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml b/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
index 91ba52b..3688282 100644
--- a/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
+++ b/k8s/providers/hetzner/infrastructure-controllers-coroot/kustomization.yaml
@@ -5,7 +5,6 @@ resources:
   - ../infrastructure/controllers/
   - ../../../bases/infrastructure/controllers/coroot/
 components:
-  - ../../../components/observability-coroot/disable-watchdog/
   - ../../../components/observability-coroot/remove-controllers-loki-alloy/
   - ../../../components/observability-coroot/remove-controller-opencost/
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 801cca6..ffdd2c8 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -155,7 +155,7 @@ assert_coroot_heartbeat_contract() {
         select($container.args == [
           "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"$1\" || true",
           "cluster-heartbeat",
-          "${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}"
+          "${cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}"
         ]) |
         select($container.securityContext.allowPrivilegeEscalation == false) |
         select($container.securityContext.readOnlyRootFilesystem == true) |
@@ -228,7 +228,7 @@ assert_coroot_heartbeat_substitution() {
   local policy_matches
 
   substituted_path="${tmp_dir}/$(basename "${rendered_path}" .yaml)-heartbeat-substituted.yaml"
-  heartbeat_token+='{alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}'
+  heartbeat_token+='{cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}'
   heartbeat_host="${heartbeat_url#https://}"
   heartbeat_host="${heartbeat_host%%/*}"
   while IFS= read -r rendered_line || [[ -n "${rendered_line}" ]]; do
@@ -277,30 +277,13 @@ assert_watchdog_enabled() {
         select(.kind == "HelmRelease" and .metadata.namespace == "monitoring" and .metadata.name == "kube-prometheus-stack") |
         select(.spec.values.defaultRules.create == true) |
         select((.spec.values.defaultRules.rules.general // true) == true) |
-        select((.spec.values.defaultRules.disabled.Watchdog // false) == false)] | length'
+        select((.spec.values.defaultRules.disabled.Watchdog // false) == false) |
+        select([.spec.values.alertmanager.config.receivers[]? |
+          select(.name == "heartbeat") |
+          select(.webhook_configs[0].url == "${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}")] | length == 1)] | length'
   )"
   if [[ "${matches}" != "1" ]]; then
-    echo "kube-prometheus-stack must render its enabled general/Watchdog rule in ${rendered_path}" >&2
-    return 1
-  fi
-}
-
-# Validates that a Coroot profile assigns the external heartbeat exclusively to
-# the dedicated CronJob instead of letting Watchdog reset the same monitor.
-assert_watchdog_disabled() {
-  local rendered_path="$1"
-  local matches
-
-  matches="$(
-    yq eval-all -o=json '.' "${rendered_path}" |
-      jq -s '[.[] |
-        select(.kind == "HelmRelease" and .metadata.namespace == "monitoring" and .metadata.name == "kube-prometheus-stack") |
-        select(.spec.values.defaultRules.create == true) |
-        select((.spec.values.defaultRules.rules.general // true) == true) |
-        select(.spec.values.defaultRules.disabled.Watchdog == true)] | length'
-  )"
-  if [[ "${matches}" != "1" ]]; then
-    echo "Coroot profile must disable Watchdog while cluster-heartbeat owns the external monitor in ${rendered_path}" >&2
+    echo "kube-prometheus-stack must keep Watchdog on the distinct Alertmanager pipeline monitor in ${rendered_path}" >&2
     return 1
   fi
 }
@@ -791,7 +774,7 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_coroot_sso_controller_contract "${rendered_path}"
   assert_coroot_heartbeat_contract "${rendered_path}"
   assert_coroot_heartbeat_substitution "${rendered_path}"
-  assert_watchdog_disabled "${rendered_path}"
+  assert_watchdog_enabled "${rendered_path}"
 done
 
 for rendered_path in "${docker_infrastructure}" "${hetzner_infrastructure}"; do
@@ -956,13 +939,13 @@ for documented_boundary in \
   "removes OpenCost and its Headlamp" \
   "retires the legacy Loki and Alloy log path" \
   "stages one hardened \`cluster-heartbeat\` CronJob" \
-  "reuses the existing encrypted heartbeat URL" \
+  "uses a separate optional \`CLUSTER_HEARTBEAT_URL\` secret" \
+  "Never point both inputs at the same monitor" \
   "permits only \`hc-ping.com:443\`" \
-  "runs at platform-critical priority" \
-  "The CronJob exclusively owns" \
-  "disables Watchdog" \
-  "Default profiles keep Watchdog unchanged" \
-  "does not add a new secret" \
+  "bootstrap rejects any other configured host" \
+  "platform-critical priority" \
+  "keep Watchdog unchanged" \
+  "preserves a dead-man signal" \
   "keeps kube-prometheus-stack" \
   "Cost allocation is therefore unavailable" \
   "reuses the existing encrypted webhook" \
@@ -991,11 +974,13 @@ fi
 
 alerting_guide="${repo_root}/docs/dr/alerting.md"
 for documented_heartbeat_boundary in \
-  "The default profile keeps the \`Watchdog\` alert" \
-  "The Coroot profile gives the existing URL exclusively" \
+  "Two independent dead-man checks" \
+  "never give them the same ping URL" \
+  "Watchdog stays enabled" \
+  "cluster_heartbeat_url" \
   "only \`hc-ping.com:443\`" \
-  "disables Watchdog" \
-  "Watchdog path stays unchanged" \
+  "bootstrap rejects another configured host" \
+  "Keeping Watchdog active also preserves" \
   "Flux reconciliation alerting still depends on kube-prometheus-stack"; do
   if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then
     echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2
@@ -1003,6 +988,23 @@ for documented_heartbeat_boundary in \
   fi
 done
 
+bootstrap_workflow="${repo_root}/.github/workflows/bootstrap.yaml"
+for bootstrap_boundary in \
+  "CLUSTER_HEARTBEAT_URL: \${{ secrets.CLUSTER_HEARTBEAT_URL }}" \
+  '""|https://hc-ping.com/*) ;;' \
+  'CLUSTER_HEARTBEAT_URL must use https://hc-ping.com/ or be unset' \
+  '.stringData.cluster_heartbeat_url = strenv(CLUSTER_HEARTBEAT_URL)'; do
+  if ! grep -Fq "${bootstrap_boundary}" "${bootstrap_workflow}"; then
+    echo "bootstrap workflow does not retain cluster-heartbeat boundary: ${bootstrap_boundary}" >&2
+    exit 1
+  fi
+done
+
+if grep -Fq 'alertmanager_heartbeat_url' "${repo_root}/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml"; then
+  echo "Coroot CronJob must not share the Alertmanager pipeline monitor" >&2
+  exit 1
+fi
+
 if grep -R -E -n '(devantler|homelab|hooks\.slack\.com|hcloud|alertmanager_webhook_url)' \
   "${repo_root}/k8s/bases/infrastructure/controllers/coroot" \
   "${repo_root}/k8s/bases/infrastructure/coroot" \

From b86b89f095e5a0fc1785a4a2aa5572f75313066d Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 04:33:54 +0200
Subject: [PATCH 22/26] fix(observability): protect heartbeat credentials

---
 .github/workflows/bootstrap.yaml              |  5 ++
 docs/TEMPLATING.md                            |  2 +
 docs/dr/alerting.md                           | 14 ++--
 .../controllers/coroot/cron-job.yaml          | 17 ++--
 .../controllers/coroot/kustomization.yaml     |  1 +
 .../controllers/coroot/secret-heartbeat.yaml  | 10 +++
 scripts/validate-observability-option.sh      | 79 ++++++++++++++-----
 7 files changed, 98 insertions(+), 30 deletions(-)
 create mode 100644 k8s/bases/infrastructure/controllers/coroot/secret-heartbeat.yaml

diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml
index 612e61b..99b1052 100644
--- a/.github/workflows/bootstrap.yaml
+++ b/.github/workflows/bootstrap.yaml
@@ -206,6 +206,11 @@ jobs:
               exit 1
               ;;
           esac
+          if [ -n "${CLUSTER_HEARTBEAT_URL}" ] && \
+             [ "${CLUSTER_HEARTBEAT_URL}" = "${ALERTMANAGER_HEARTBEAT_URL}" ]; then
+            echo "::error::CLUSTER_HEARTBEAT_URL must use a different external check than ALERTMANAGER_HEARTBEAT_URL"
+            exit 1
+          fi
 
           fill_base() { # variables-base-secret
             yq -i '
diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index 5bf0dcb..e29903a 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -94,6 +94,8 @@ the existing Alertmanager Watchdog check keeps proving the Prometheus-to-Alertma
 pipeline. Never point both inputs at the same monitor. The CronJob runs at
 platform-critical priority, and its dedicated policy permits only `hc-ping.com:443`
 plus that hostname's exact DNS query; bootstrap rejects any other configured host.
+The substituted bearer URL lives in a Kubernetes Secret and reaches the container
+through a Secret-backed environment variable, not the PodSpec command or arguments.
 The Coroot infrastructure
 overlay alone narrows the generated namespace policies around the CronJob's
 purpose-specific label, so default profiles retain their global selectors. The
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index ca54d4c..4763b1b 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -137,10 +137,11 @@ Two sources:
 
 ## Per-environment setup (manual SOPS steps)
 
-The Slack webhook and heartbeat URL are secrets, so they live in the
-per-cluster `variables-cluster-secret.enc.yaml` (under `bootstrap/`) and
-must be set by hand. Both are read from the `Secret` `variables-cluster`,
-which is a Flux `substituteFrom` source.
+The three secret inputs — `alertmanager_webhook_url`,
+`alertmanager_heartbeat_url`, and `cluster_heartbeat_url` — live in the
+per-cluster `variables-cluster-secret.enc.yaml` (under `bootstrap/`) and must
+be set by hand. All three are read from the `Secret` `variables-cluster`, which
+is a Flux `substituteFrom` source.
 
 ```bash
 # 1. Slack incoming webhook for alert notifications.
@@ -158,8 +159,9 @@ sops --set '["stringData"]["cluster_heartbeat_url"] "https://hc-ping.com/<differ
 
 Slack side: create an incoming webhook for your alerts channel (e.g.
 `#platform-alerts`) — the channel in the config is cosmetic, since an incoming
-webhook posts to the channel it was created for. healthchecks.io side: create
-the check, connect its Slack integration, copy the ping URL.
+webhook posts to the channel it was created for. On healthchecks.io, create two
+distinct checks, connect each check's Slack integration, and copy their different
+ping URLs into `alertmanager_heartbeat_url` and `cluster_heartbeat_url` respectively.
 
 | Env   | `alertmanager_webhook_url`      | `alertmanager_heartbeat_url`       | `cluster_heartbeat_url`                 |
 | ----- | ------------------------------- | ---------------------------------- | --------------------------------------- |
diff --git a/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml b/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
index eb4e584..a777dd8 100644
--- a/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
@@ -7,9 +7,10 @@
 # coupling this signal to Coroot would turn a component outage into a false
 # cluster-down alarm.
 #
-# The URL comes from the existing Flux substitution input. Local and
-# unconfigured clusters use the invalid fallback, and `|| true` keeps the Job
-# from flapping when no external monitor is configured. The signal is the
+# The URL comes from the dedicated Flux substitution input through a Secret
+# reference, so the bearer token does not appear in the PodSpec or argv. Local
+# and unconfigured clusters use the invalid fallback, and `|| true` keeps the
+# Job from flapping when no external monitor is configured. The signal is the
 # absence of pings at the external monitor, not the Job exit code.
 apiVersion: batch/v1
 kind: CronJob
@@ -50,9 +51,13 @@ spec:
                 - >-
                   curl -sf --max-time 10
                   --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused
-                  "$1" || true
-                - cluster-heartbeat
-                - "${cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}"
+                  "$HEARTBEAT_URL" || true
+              env:
+                - name: HEARTBEAT_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: cluster-heartbeat
+                      key: url
               securityContext:
                 allowPrivilegeEscalation: false
                 readOnlyRootFilesystem: true
diff --git a/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml b/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
index a1d40a5..3d551e6 100644
--- a/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
@@ -8,6 +8,7 @@ resources:
   - cilium-network-policy-heartbeat.yaml
   - cilium-network-policy.yaml
   - cron-job.yaml
+  - secret-heartbeat.yaml
   - helm-repository.yaml
   - helm-release.yaml
   - http-route.yaml
diff --git a/k8s/bases/infrastructure/controllers/coroot/secret-heartbeat.yaml b/k8s/bases/infrastructure/controllers/coroot/secret-heartbeat.yaml
new file mode 100644
index 0000000..7b9cf3f
--- /dev/null
+++ b/k8s/bases/infrastructure/controllers/coroot/secret-heartbeat.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cluster-heartbeat
+  namespace: observability
+type: Opaque
+stringData:
+  # Flux substitutes the encrypted cluster variable here. The CronJob reads the
+  # resulting Secret so the bearer URL never appears in its PodSpec or argv.
+  url: "${cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}"
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index ffdd2c8..1e34137 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -111,6 +111,7 @@ assert_default_off() {
   assert_resource_count "${rendered_path}" Coroot observability coroot 0
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
   assert_resource_count "${rendered_path}" CronJob observability cluster-heartbeat 0
+  assert_resource_count "${rendered_path}" Secret observability cluster-heartbeat 0
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 0
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 0
   assert_resource_count "${rendered_path}" HTTPRoute observability coroot 0
@@ -122,9 +123,10 @@ assert_coroot_heartbeat_contract() {
   local rendered_path="$1"
   local heartbeat_count
   local cronjob_contract
+  local secret_contract
   local heartbeat_egress_contract
   local coroot_selector_contract
-  local broad_external_egress
+  local heartbeat_selecting_policy_count
 
   heartbeat_count="$(resource_count "${rendered_path}" CronJob observability cluster-heartbeat)"
   cronjob_contract="$(
@@ -153,10 +155,12 @@ assert_coroot_heartbeat_contract() {
           "-c"
         ]) |
         select($container.args == [
-          "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"$1\" || true",
-          "cluster-heartbeat",
-          "${cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}"
+          "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"$HEARTBEAT_URL\" || true"
         ]) |
+        select($container.env == [{
+          "name":"HEARTBEAT_URL",
+          "valueFrom":{"secretKeyRef":{"name":"cluster-heartbeat","key":"url"}}
+        }]) |
         select($container.securityContext.allowPrivilegeEscalation == false) |
         select($container.securityContext.readOnlyRootFilesystem == true) |
         select($container.securityContext.runAsNonRoot == true and $container.securityContext.runAsUser == 65532 and $container.securityContext.runAsGroup == 65532) |
@@ -166,6 +170,13 @@ assert_coroot_heartbeat_contract() {
           "limits":{"cpu":"50m","memory":"32Mi"}
         })] | length'
   )"
+  secret_contract="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s '[.[] |
+        select(.kind == "Secret" and .metadata.namespace == "observability" and .metadata.name == "cluster-heartbeat") |
+        select(.type == "Opaque") |
+        select(.stringData == {"url":"${cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}"})] | length'
+  )"
   heartbeat_egress_contract="$(
     yq eval-all -o=json '.' "${rendered_path}" |
       jq -s '[.[] |
@@ -196,20 +207,38 @@ assert_coroot_heartbeat_contract() {
         select(.spec.endpointSelector.matchExpressions == [{"key":"platform-heartbeat","operator":"NotIn","values":["true"]}]) |
         select(all(.spec.egress[]?.toFQDNs[]?; .matchName != "hc-ping.com"))] | length'
   )"
-  broad_external_egress="$(
+  heartbeat_selecting_policy_count="$(
     yq eval-all -o=json '.' "${rendered_path}" |
-      jq -s '[.[] |
-        select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-coroot") |
-        .spec.egress[]? |
-        select(any(.toEntities[]?; . == "world" or . == "all") or any(.toFQDNs[]?; has("matchPattern")))] | length'
+      jq -s '
+        def selector_matches($labels):
+          (.spec.endpointSelector // {}) as $selector |
+          all(($selector.matchLabels // {} | to_entries[]); $labels[.key] == .value) and
+          all(($selector.matchExpressions // [])[];
+            .key as $key |
+            ($labels[$key] // null) as $value |
+            if .operator == "In" then
+              $value != null and ((.values // []) | index($value) != null)
+            elif .operator == "NotIn" then
+              $value == null or ((.values // []) | index($value) == null)
+            elif .operator == "Exists" then
+              $value != null
+            elif .operator == "DoesNotExist" then
+              $value == null
+            else
+              false
+            end);
+        {"app":"cluster-heartbeat","platform-heartbeat":"true"} as $heartbeat_labels |
+        [.[] |
+          select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability") |
+          select(selector_matches($heartbeat_labels))] | length'
   )"
 
-  if [[ "${heartbeat_count}" != "1" || "${cronjob_contract}" != "1" ]]; then
-    echo "Coroot profile must render one hardened five-minute cluster heartbeat in ${rendered_path}" >&2
+  if [[ "${heartbeat_count}" != "1" || "${cronjob_contract}" != "1" || "${secret_contract}" != "1" ]]; then
+    echo "Coroot profile must render one hardened five-minute heartbeat with one Secret reference in ${rendered_path}" >&2
     return 1
   fi
-  if [[ "${heartbeat_egress_contract}" != "1" || "${coroot_selector_contract}" != "1" || "${broad_external_egress}" != "0" ]]; then
-    echo "Coroot heartbeat must be excluded from broad policies and have one exact configured-host egress path without wildcard or world access in ${rendered_path}" >&2
+  if [[ "${heartbeat_egress_contract}" != "1" || "${coroot_selector_contract}" != "1" || "${heartbeat_selecting_policy_count}" != "1" ]]; then
+    echo "Coroot heartbeat must match only its one exact configured-host egress policy in ${rendered_path}" >&2
     return 1
   fi
 }
@@ -225,6 +254,7 @@ assert_coroot_heartbeat_substitution() {
   local rendered_prefix
   local rendered_suffix
   local cronjob_matches
+  local secret_matches
   local policy_matches
 
   substituted_path="${tmp_dir}/$(basename "${rendered_path}" .yaml)-heartbeat-substituted.yaml"
@@ -247,10 +277,18 @@ assert_coroot_heartbeat_substitution() {
         .spec.jobTemplate.spec.template.spec.containers[0] as $container |
         select($container.command == ["/bin/sh", "-c"]) |
         select($container.args == [
-          "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"$1\" || true",
-          "cluster-heartbeat",
-          $heartbeat_url
-        ])] | length'
+          "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"$HEARTBEAT_URL\" || true"
+        ]) |
+        select($container.env == [{
+          "name":"HEARTBEAT_URL",
+          "valueFrom":{"secretKeyRef":{"name":"cluster-heartbeat","key":"url"}}
+        }])] | length'
+  )"
+  secret_matches="$(
+    yq eval-all -o=json '.' "${substituted_path}" |
+      jq -s --arg heartbeat_url "${heartbeat_url}" '[.[] |
+        select(.kind == "Secret" and .metadata.namespace == "observability" and .metadata.name == "cluster-heartbeat") |
+        select(.stringData.url == $heartbeat_url)] | length'
   )"
   policy_matches="$(
     yq eval-all -o=json '.' "${substituted_path}" |
@@ -260,7 +298,7 @@ assert_coroot_heartbeat_substitution() {
         select(any(.spec.egress[]?.toPorts[]?.rules.dns[]?; .matchName == $heartbeat_host))] | length'
   )"
 
-  if [[ "${cronjob_matches}" != "1" || "${policy_matches}" != "1" ]]; then
+  if [[ "${cronjob_matches}" != "1" || "${secret_matches}" != "1" || "${policy_matches}" != "1" ]]; then
     echo "Flux-substituted heartbeat URL and exact policy host must agree in ${rendered_path}" >&2
     return 1
   fi
@@ -768,6 +806,7 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_resource_count "${rendered_path}" Coroot observability coroot 0
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
   assert_resource_count "${rendered_path}" HelmRelease monitoring kube-prometheus-stack 1
+  assert_resource_count "${rendered_path}" Secret observability cluster-heartbeat 1
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 1
   assert_opencost_absent "${rendered_path}"
@@ -943,6 +982,7 @@ for documented_boundary in \
   "Never point both inputs at the same monitor" \
   "permits only \`hc-ping.com:443\`" \
   "bootstrap rejects any other configured host" \
+  "Secret-backed environment variable" \
   "platform-critical priority" \
   "keep Watchdog unchanged" \
   "preserves a dead-man signal" \
@@ -978,6 +1018,7 @@ for documented_heartbeat_boundary in \
   "never give them the same ping URL" \
   "Watchdog stays enabled" \
   "cluster_heartbeat_url" \
+  "The three secret inputs" \
   "only \`hc-ping.com:443\`" \
   "bootstrap rejects another configured host" \
   "Keeping Watchdog active also preserves" \
@@ -993,6 +1034,8 @@ for bootstrap_boundary in \
   "CLUSTER_HEARTBEAT_URL: \${{ secrets.CLUSTER_HEARTBEAT_URL }}" \
   '""|https://hc-ping.com/*) ;;' \
   'CLUSTER_HEARTBEAT_URL must use https://hc-ping.com/ or be unset' \
+  "[ \"\${CLUSTER_HEARTBEAT_URL}\" = \"\${ALERTMANAGER_HEARTBEAT_URL}\" ]; then" \
+  'CLUSTER_HEARTBEAT_URL must use a different external check than ALERTMANAGER_HEARTBEAT_URL' \
   '.stringData.cluster_heartbeat_url = strenv(CLUSTER_HEARTBEAT_URL)'; do
   if ! grep -Fq "${bootstrap_boundary}" "${bootstrap_workflow}"; then
     echo "bootstrap workflow does not retain cluster-heartbeat boundary: ${bootstrap_boundary}" >&2

From d80f2d12c3371e04336bb667710be8dc133a4b9a Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 04:38:00 +0200
Subject: [PATCH 23/26] docs(observability): describe validator helpers

---
 scripts/validate-observability-option.sh | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 1e34137..cd776f6 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -32,6 +32,7 @@ resource_count() {
       '[.[] | select(type == "object" and .kind == $kind and (.metadata.namespace // "") == $namespace and .metadata.name == $name)] | length'
 }
 
+# Counts rendered resources owned by the OpenCost namespace.
 opencost_resource_count() {
   local rendered_path="$1"
 
@@ -39,6 +40,7 @@ opencost_resource_count() {
     jq -s '[.[] | select(type == "object" and ((.kind == "Namespace" and .metadata.name == "opencost") or .metadata.namespace == "opencost"))] | length'
 }
 
+# Counts rendered resources whose serialized content still references OpenCost.
 opencost_reference_count() {
   local rendered_path="$1"
 
@@ -46,6 +48,7 @@ opencost_reference_count() {
     jq -s '[.[] | select(type == "object" and (tojson | test("opencost"; "i")))] | length'
 }
 
+# Counts Grafana data sources that target the retired Loki service.
 loki_datasource_count() {
   local rendered_path="$1"
 
@@ -56,6 +59,7 @@ loki_datasource_count() {
       select(.type == "loki" or ((.url // "") | test("loki\\.monitoring\\.svc")))] | length'
 }
 
+# Counts rendered resources whose serialized content still references Loki.
 loki_service_reference_count() {
   local rendered_path="$1"
 
@@ -79,6 +83,7 @@ assert_resource_count() {
   fi
 }
 
+# Returns the source path for one uniquely rendered Flux Kustomization.
 flux_path() {
   local rendered_path="$1"
   local name="$2"
@@ -90,6 +95,7 @@ flux_path() {
        if length == 1 then .[0].spec.path else "invalid-count:\(length)" end'
 }
 
+# Fails when a Flux Kustomization does not point at the expected profile path.
 assert_flux_path() {
   local rendered_path="$1"
   local name="$2"
@@ -419,6 +425,7 @@ assert_heartbeat_policy_exclusion_absent() {
   fi
 }
 
+# Verifies that a default controller profile retains the complete OpenCost surface.
 assert_opencost_present() {
   local rendered_path="$1"
 
@@ -429,6 +436,7 @@ assert_opencost_present() {
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy opencost allow-opencost 1
 }
 
+# Verifies that a rendered profile contains no OpenCost-owned resources.
 assert_opencost_resources_absent() {
   local rendered_path="$1"
   local actual
@@ -440,6 +448,7 @@ assert_opencost_resources_absent() {
   fi
 }
 
+# Verifies the exact number of rendered resources that reference OpenCost.
 assert_opencost_reference_count() {
   local rendered_path="$1"
   local expected="$2"
@@ -452,6 +461,7 @@ assert_opencost_reference_count() {
   fi
 }
 
+# Verifies that both OpenCost resources and serialized references are absent.
 assert_opencost_absent() {
   local rendered_path="$1"
   local actual
@@ -464,6 +474,7 @@ assert_opencost_absent() {
   fi
 }
 
+# Verifies that Coroot reuses the hardened auth proxy without weakening it.
 assert_auth_proxy_with_coroot() {
   local default_rendered_path="$1"
   local opt_in_rendered_path="$2"
@@ -497,6 +508,7 @@ assert_auth_proxy_with_coroot() {
   fi
 }
 
+# Verifies the Coroot controller profile's SSO and access-control contract.
 assert_coroot_sso_controller_contract() {
   local rendered_path="$1"
   local route_contract
@@ -569,6 +581,7 @@ assert_coroot_sso_controller_contract() {
   fi
 }
 
+# Verifies whether a rendered Coroot project grants the expected admin role.
 assert_coroot_admin_role() {
   local rendered_path="$1"
   local expected="$2"
@@ -593,6 +606,7 @@ assert_coroot_admin_role() {
   fi
 }
 
+# Counts policy rules that omit the required namespace exclusion.
 policy_rules_without_namespace_exclusion() {
   local rendered_path="$1"
   local policy_name="$2"
@@ -611,6 +625,7 @@ policy_rules_without_namespace_exclusion() {
        end'
 }
 
+# Counts policy rules that carry a namespace exclusion.
 policy_rules_with_namespace_exclusion() {
   local rendered_path="$1"
   local policy_name="$2"
@@ -629,6 +644,7 @@ policy_rules_with_namespace_exclusion() {
        end'
 }
 
+# Verifies that every rule in a policy excludes the selected namespace.
 assert_namespace_excluded_from_all_policy_rules() {
   local rendered_path="$1"
   local policy_name="$2"
@@ -642,6 +658,7 @@ assert_namespace_excluded_from_all_policy_rules() {
   fi
 }
 
+# Verifies that no rule in a policy excludes the selected namespace.
 assert_namespace_not_excluded_from_policy() {
   local rendered_path="$1"
   local policy_name="$2"
@@ -655,6 +672,7 @@ assert_namespace_not_excluded_from_policy() {
   fi
 }
 
+# Verifies the exact namespace match count in one security exception.
 assert_security_exception_namespace_count() {
   local rendered_path="$1"
   local exception_name="$2"

From 06fc41ffed6cecfdf054de7be9b3ef65693290aa Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 05:47:12 +0200
Subject: [PATCH 24/26] fix(observability): close heartbeat review gaps

---
 .github/workflows/bootstrap.yaml              |  28 ++-
 docs/BOOTSTRAP.md                             |   2 +-
 docs/TEMPLATING.md                            |  20 ++-
 docs/dr/alerting.md                           |  17 +-
 .../cilium-network-policy.yaml}               |   0
 .../cluster-heartbeat}/cron-job.yaml          |   0
 .../apps/cluster-heartbeat/kustomization.yaml |   9 +
 .../cluster-heartbeat/secret.yaml}            |   0
 .../controllers/coroot/kustomization.yaml     |   3 -
 .../docker/apps-coroot/kustomization.yaml     |   1 +
 .../hetzner/apps-coroot/kustomization.yaml    |   1 +
 scripts/validate-observability-option.sh      | 166 ++++++++++++++++--
 12 files changed, 205 insertions(+), 42 deletions(-)
 rename k8s/bases/{infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml => apps/cluster-heartbeat/cilium-network-policy.yaml} (100%)
 rename k8s/bases/{infrastructure/controllers/coroot => apps/cluster-heartbeat}/cron-job.yaml (100%)
 create mode 100644 k8s/bases/apps/cluster-heartbeat/kustomization.yaml
 rename k8s/bases/{infrastructure/controllers/coroot/secret-heartbeat.yaml => apps/cluster-heartbeat/secret.yaml} (100%)

diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml
index 99b1052..ea3edc6 100644
--- a/.github/workflows/bootstrap.yaml
+++ b/.github/workflows/bootstrap.yaml
@@ -179,6 +179,7 @@ jobs:
           grep -rIl 'example-org' k8s/ 2>/dev/null | xargs -r sed -i "s|example-org|${OWNER}|g"
 
       - name: 🔐 Encrypt secrets into *.enc.yaml
+        shell: bash
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
@@ -199,15 +200,26 @@ jobs:
           # policy. Reject a configured URL that the cluster cannot reach rather
           # than encrypting a silently dead monitor. Empty stays valid so default
           # profiles and operators who have not opted into Coroot remain inert.
-          case "${CLUSTER_HEARTBEAT_URL}" in
-            ""|https://hc-ping.com/*) ;;
-            *)
-              echo "::error::CLUSTER_HEARTBEAT_URL must use https://hc-ping.com/ or be unset"
+          heartbeat_url_pattern='^https://hc-ping\.com/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$'
+          cluster_heartbeat_check_id=""
+          alertmanager_heartbeat_check_id=""
+          alertmanager_heartbeat_url_lower="$(printf '%s' "${ALERTMANAGER_HEARTBEAT_URL}" | tr '[:upper:]' '[:lower:]')"
+          if [ -n "${CLUSTER_HEARTBEAT_URL}" ]; then
+            if [[ ! "${CLUSTER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then
+              echo "::error::CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/<lowercase-uuid> URL or be unset"
               exit 1
-              ;;
-          esac
-          if [ -n "${CLUSTER_HEARTBEAT_URL}" ] && \
-             [ "${CLUSTER_HEARTBEAT_URL}" = "${ALERTMANAGER_HEARTBEAT_URL}" ]; then
+            fi
+            cluster_heartbeat_check_id="${BASH_REMATCH[1]}"
+          fi
+          if [[ "${alertmanager_heartbeat_url_lower}" == *://hc-ping.com* ]]; then
+            if [[ ! "${ALERTMANAGER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then
+              echo "::error::ALERTMANAGER_HEARTBEAT_URL must be canonical when it uses hc-ping.com"
+              exit 1
+            fi
+            alertmanager_heartbeat_check_id="${BASH_REMATCH[1]}"
+          fi
+          if [ -n "${cluster_heartbeat_check_id}" ] && \
+             [ "${cluster_heartbeat_check_id}" = "${alertmanager_heartbeat_check_id}" ]; then
             echo "::error::CLUSTER_HEARTBEAT_URL must use a different external check than ALERTMANAGER_HEARTBEAT_URL"
             exit 1
           fi
diff --git a/docs/BOOTSTRAP.md b/docs/BOOTSTRAP.md
index acf014a..d91e4d4 100644
--- a/docs/BOOTSTRAP.md
+++ b/docs/BOOTSTRAP.md
@@ -112,7 +112,7 @@ Set these in your **new instance repository** under *Settings → Secrets and va
 | `R2_SECRET_ACCESS_KEY` | R2 (S3) secret access key for backups | Same place as the access key ID. |
 | `ALERTMANAGER_WEBHOOK_URL` | Incoming webhook for alert notifications (e.g. Slack) | Your chat platform’s incoming-webhook setup. |
 | `ALERTMANAGER_HEARTBEAT_URL` | Ping URL that proves the Prometheus → Alertmanager pipeline is alive | healthchecks.io (or similar) check ping URL. |
-| `CLUSTER_HEARTBEAT_URL` | Optional Coroot-profile cluster-liveness ping URL | A **separate** `https://hc-ping.com/...` check; never reuse the Alertmanager check. |
+| `CLUSTER_HEARTBEAT_URL` | Optional Coroot-profile cluster-liveness ping URL | A **separate**, canonical `https://hc-ping.com/<lowercase-uuid>` URL; never reuse the Alertmanager check under another URL spelling. |
 | `GITHUB_APP_CLIENT_SECRET` | **Client secret** of the *cluster SSO* GitHub App (Dex’s GitHub connector) | The SSO App’s settings page (generate a client secret). |
 | `APP_PRIVATE_KEY` | Private key (`.pem`) of the **bootstrap** GitHub App | Generated when you created the bootstrap App. |
 
diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index e29903a..abe6463 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -93,16 +93,20 @@ dead-man signal. It uses a separate optional `CLUSTER_HEARTBEAT_URL` secret so
 the existing Alertmanager Watchdog check keeps proving the Prometheus-to-Alertmanager
 pipeline. Never point both inputs at the same monitor. The CronJob runs at
 platform-critical priority, and its dedicated policy permits only `hc-ping.com:443`
-plus that hostname's exact DNS query; bootstrap rejects any other configured host.
+plus that hostname's exact DNS query. Bootstrap accepts only a canonical
+`https://hc-ping.com/<lowercase-uuid>` URL and compares check identities, so
+fragments, query strings, or trailing slashes cannot make both inputs target the
+same external check under different spellings.
 The substituted bearer URL lives in a Kubernetes Secret and reaches the container
 through a Secret-backed environment variable, not the PodSpec command or arguments.
-The Coroot infrastructure
-overlay alone narrows the generated namespace policies around the CronJob's
-purpose-specific label, so default profiles retain their global selectors. The
-invalid URL fallback keeps local and unconfigured instances inert. Both default
-and Coroot profiles keep Watchdog unchanged on the distinct Alertmanager pipeline
-monitor. This also preserves a dead-man signal if the later Coroot infrastructure
-layer has not yet established the CronJob's generated-policy isolation.
+The Coroot infrastructure overlay alone narrows the generated namespace policies
+around the CronJob's purpose-specific label, so default profiles retain their
+global selectors. The dependent apps layer stages the heartbeat only after that
+infrastructure reconciliation is ready, preventing an existing namespace-wide
+deny policy from blocking the new workload during an upgrade. The invalid URL
+fallback keeps local and unconfigured instances inert. Both default and Coroot
+profiles keep Watchdog unchanged on the distinct Alertmanager pipeline monitor,
+which preserves a dead-man signal until the apps layer starts the direct check.
 Kube-prometheus-stack keeps owning its remaining alert
 rules, including Flux reconciliation alerts, until that migration is delivered
 separately.
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index 4763b1b..7558273 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -62,13 +62,16 @@ The Coroot profile adds a dedicated `cluster-heartbeat` CronJob, which pings a
 reuses `alertmanager_heartbeat_url`: doing so would let the CronJob keep that
 monitor green while the Prometheus-to-Alertmanager pipeline was down.
 Watchdog stays enabled on its existing check in both default and Coroot profiles.
-The CronJob's separate `cluster_heartbeat_url` accepts only `https://hc-ping.com/...`;
-its network policy allows only `hc-ping.com:443` plus that hostname's exact DNS
-query, and bootstrap rejects another configured host. The Coroot overlay alone
-narrows the namespace deny and DNS policies around this purpose-labelled workload;
-default profiles keep their global selectors. Keeping Watchdog active also preserves
-the original dead-man path if the later infrastructure reconciliation has not yet
-established that isolation. An invalid fallback keeps an unconfigured CronJob quiet.
+The CronJob's separate `cluster_heartbeat_url` accepts only a canonical
+`https://hc-ping.com/<lowercase-uuid>` URL; its network policy allows only
+`hc-ping.com:443` plus that hostname's exact DNS query. Bootstrap compares
+canonical check identities, so URL fragments, query strings, and trailing slashes
+cannot disguise reuse of the Alertmanager check. The Coroot infrastructure overlay
+alone narrows the namespace deny and DNS policies around this purpose-labelled
+workload; default profiles keep their global selectors. The dependent apps layer
+stages the CronJob only after that exclusion is ready, while Watchdog preserves the
+original dead-man path until the apps reconciliation completes. An invalid fallback
+keeps an unconfigured CronJob quiet.
 Flux reconciliation alerting still depends on kube-prometheus-stack, so that stack
 remains transitional until a later slice replaces those rules before removing it.
 
diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml b/k8s/bases/apps/cluster-heartbeat/cilium-network-policy.yaml
similarity index 100%
rename from k8s/bases/infrastructure/controllers/coroot/cilium-network-policy-heartbeat.yaml
rename to k8s/bases/apps/cluster-heartbeat/cilium-network-policy.yaml
diff --git a/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml b/k8s/bases/apps/cluster-heartbeat/cron-job.yaml
similarity index 100%
rename from k8s/bases/infrastructure/controllers/coroot/cron-job.yaml
rename to k8s/bases/apps/cluster-heartbeat/cron-job.yaml
diff --git a/k8s/bases/apps/cluster-heartbeat/kustomization.yaml b/k8s/bases/apps/cluster-heartbeat/kustomization.yaml
new file mode 100644
index 0000000..75a3a4f
--- /dev/null
+++ b/k8s/bases/apps/cluster-heartbeat/kustomization.yaml
@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# The apps Flux layer depends on infrastructure. Keeping the heartbeat here
+# ensures the scoped observability deny-policy replacement is ready before the
+# purpose-labelled workload starts during an existing-cluster upgrade.
+resources:
+  - cilium-network-policy.yaml
+  - cron-job.yaml
+  - secret.yaml
diff --git a/k8s/bases/infrastructure/controllers/coroot/secret-heartbeat.yaml b/k8s/bases/apps/cluster-heartbeat/secret.yaml
similarity index 100%
rename from k8s/bases/infrastructure/controllers/coroot/secret-heartbeat.yaml
rename to k8s/bases/apps/cluster-heartbeat/secret.yaml
diff --git a/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml b/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
index 3d551e6..59132d1 100644
--- a/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
+++ b/k8s/bases/infrastructure/controllers/coroot/kustomization.yaml
@@ -5,10 +5,7 @@ kind: Kustomization
 # establish the CRD before Flux server-side dry-runs the custom resource.
 resources:
   - namespace.yaml
-  - cilium-network-policy-heartbeat.yaml
   - cilium-network-policy.yaml
-  - cron-job.yaml
-  - secret-heartbeat.yaml
   - helm-repository.yaml
   - helm-release.yaml
   - http-route.yaml
diff --git a/k8s/providers/docker/apps-coroot/kustomization.yaml b/k8s/providers/docker/apps-coroot/kustomization.yaml
index 03fb25b..f333d52 100644
--- a/k8s/providers/docker/apps-coroot/kustomization.yaml
+++ b/k8s/providers/docker/apps-coroot/kustomization.yaml
@@ -3,5 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../apps/
+  - ../../../bases/apps/cluster-heartbeat/
 components:
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/k8s/providers/hetzner/apps-coroot/kustomization.yaml b/k8s/providers/hetzner/apps-coroot/kustomization.yaml
index 03fb25b..f333d52 100644
--- a/k8s/providers/hetzner/apps-coroot/kustomization.yaml
+++ b/k8s/providers/hetzner/apps-coroot/kustomization.yaml
@@ -3,5 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ../apps/
+  - ../../../bases/apps/cluster-heartbeat/
 components:
   - ../../../components/observability-coroot/remove-opencost-references/
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index cd776f6..2c3fafd 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -109,6 +109,27 @@ assert_flux_path() {
   fi
 }
 
+# Verifies the reconciliation edge and wait boundary that keep a later Flux
+# payload from starting before its prerequisite reports ready.
+assert_flux_dependency() {
+  local rendered_path="$1"
+  local name="$2"
+  local dependency="$3"
+  local matches
+
+  matches="$(
+    yq eval-all -o=json '.' "${rendered_path}" |
+      jq -s --arg name "${name}" --arg dependency "${dependency}" '[.[] |
+        select(.kind == "Kustomization" and .metadata.namespace == "flux-system" and .metadata.name == $name) |
+        select(.spec.dependsOn == [{"name":$dependency}]) |
+        select(.spec.wait == true)] | length'
+  )"
+  if [[ "${matches}" != "1" ]]; then
+    echo "Flux Kustomization/${name} must wait for ${dependency} in ${rendered_path}" >&2
+    return 1
+  fi
+}
+
 # Verifies that the default profiles render no Coroot-owned resources.
 assert_default_off() {
   local rendered_path="$1"
@@ -123,10 +144,103 @@ assert_default_off() {
   assert_resource_count "${rendered_path}" HTTPRoute observability coroot 0
 }
 
+# Verifies that the heartbeat workload reconciles only in the apps layer,
+# after infrastructure has replaced the observability namespace's generated
+# deny policies with the scoped selectors.
+assert_heartbeat_staged_after_policy_exclusion() {
+  local controllers_rendered_path="$1"
+  local apps_rendered_path="$2"
+
+  assert_resource_count "${controllers_rendered_path}" CronJob observability cluster-heartbeat 0
+  assert_resource_count "${controllers_rendered_path}" Secret observability cluster-heartbeat 0
+  assert_resource_count "${controllers_rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 0
+  assert_resource_count "${apps_rendered_path}" CronJob observability cluster-heartbeat 1
+  assert_resource_count "${apps_rendered_path}" Secret observability cluster-heartbeat 1
+  assert_resource_count "${apps_rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 1
+}
+
+# Executes the workflow's real heartbeat preflight with bounded fixture URLs.
+run_bootstrap_heartbeat_preflight() {
+  local bootstrap_workflow="$1"
+  local cluster_heartbeat_url="$2"
+  local alertmanager_heartbeat_url="$3"
+  local validation_script
+
+  validation_script="$(
+    sed -n '/# The dedicated Coroot heartbeat has an exact hc-ping.com egress/,/^[[:space:]]*fill_base()/p' "${bootstrap_workflow}" |
+      sed '$d'
+  )"
+  env \
+    CLUSTER_HEARTBEAT_URL="${cluster_heartbeat_url}" \
+    ALERTMANAGER_HEARTBEAT_URL="${alertmanager_heartbeat_url}" \
+    bash -c "set -euo pipefail
+${validation_script}" >/dev/null 2>&1
+}
+
+# Proves that bootstrap accepts two canonical checks while rejecting aliases,
+# alternate healthchecks.io operations, and noncanonical check identifiers.
+assert_bootstrap_heartbeat_identity_contract() {
+  local bootstrap_workflow="$1"
+  local cluster_url='https://hc-ping.com/11111111-1111-4111-8111-111111111111'
+  local alertmanager_url='https://hc-ping.com/22222222-2222-4222-8222-222222222222'
+  local cluster_alias
+  local alertmanager_alias
+  local -a cluster_aliases=(
+    "${cluster_url}#cluster"
+    "${cluster_url}?source=cluster"
+    "${cluster_url}/"
+    "${cluster_url}/start"
+    "${cluster_url}/fail"
+    "${cluster_url}/log"
+    "${cluster_url}/1"
+    'https://HC-PING.com/11111111-1111-4111-8111-111111111111'
+    'https://hc-ping.com:443/11111111-1111-4111-8111-111111111111'
+    'https://hc-ping.com/project-slug/check-slug'
+    'https://hc-ping.com/AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA'
+  )
+  local -a alertmanager_aliases=(
+    "${cluster_url}"
+    "${cluster_url}#alertmanager"
+    "${cluster_url}?source=alertmanager"
+    "${cluster_url}/"
+    "${cluster_url}/start"
+    "${cluster_url}/fail"
+    "${cluster_url}/log"
+    "${cluster_url}/1"
+    'https://HC-PING.com/11111111-1111-4111-8111-111111111111'
+    'https://hc-ping.com:443/11111111-1111-4111-8111-111111111111'
+    'https://hc-ping.com/project-slug/check-slug'
+  )
+
+  if ! run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_url}" "${alertmanager_url}"; then
+    echo "bootstrap must accept two distinct canonical healthchecks.io checks" >&2
+    return 1
+  fi
+  if ! run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "" "${alertmanager_url}"; then
+    echo "bootstrap must keep an unset direct-cluster check valid" >&2
+    return 1
+  fi
+  for cluster_alias in "${cluster_aliases[@]}"; do
+    if run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_alias}" "${alertmanager_url}"; then
+      echo "bootstrap must reject every noncanonical direct-cluster check URL" >&2
+      return 1
+    fi
+  done
+  for alertmanager_alias in "${alertmanager_aliases[@]}"; do
+    if run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_url}" "${alertmanager_alias}"; then
+      echo "bootstrap must reject every alias of the direct-cluster check" >&2
+      return 1
+    fi
+  done
+}
+
+assert_bootstrap_heartbeat_identity_contract "${repo_root}/.github/workflows/bootstrap.yaml"
+
 # Validates the singleton workload, isolation selector, and exact egress contract
 # rendered by each explicit Coroot profile.
 assert_coroot_heartbeat_contract() {
   local rendered_path="$1"
+  local controllers_rendered_path="$2"
   local heartbeat_count
   local cronjob_contract
   local secret_contract
@@ -207,14 +321,14 @@ assert_coroot_heartbeat_contract() {
         })] | length'
   )"
   coroot_selector_contract="$(
-    yq eval-all -o=json '.' "${rendered_path}" |
+    yq eval-all -o=json '.' "${controllers_rendered_path}" |
       jq -s '[.[] |
         select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-coroot") |
         select(.spec.endpointSelector.matchExpressions == [{"key":"platform-heartbeat","operator":"NotIn","values":["true"]}]) |
         select(all(.spec.egress[]?.toFQDNs[]?; .matchName != "hc-ping.com"))] | length'
   )"
   heartbeat_selecting_policy_count="$(
-    yq eval-all -o=json '.' "${rendered_path}" |
+    yq eval-all -o=json '.' "${rendered_path}" "${controllers_rendered_path}" |
       jq -s '
         def selector_matches($labels):
           (.spec.endpointSelector // {}) as $selector |
@@ -744,6 +858,10 @@ assert_flux_path "${prod_cluster_coroot}" bootstrap clusters/prod/bootstrap
 assert_flux_path "${prod_cluster_coroot}" infrastructure-controllers providers/hetzner/infrastructure-controllers-coroot
 assert_flux_path "${prod_cluster_coroot}" infrastructure providers/hetzner/infrastructure-coroot
 assert_flux_path "${prod_cluster_coroot}" apps providers/hetzner/apps-coroot
+for rendered_path in "${local_cluster_coroot}" "${prod_cluster_coroot}"; do
+  assert_flux_dependency "${rendered_path}" infrastructure infrastructure-controllers
+  assert_flux_dependency "${rendered_path}" apps infrastructure
+done
 
 # Cluster renders contain Flux pointers, so inspect the provider payloads those
 # pointers reconcile. This catches accidental activation in either layer.
@@ -756,8 +874,10 @@ render k8s/providers/hetzner/apps/ "${prod_apps_default}"
 for rendered_path in \
   "${local_controllers_default}" \
   "${local_infrastructure_default}" \
+  "${local_apps_default}" \
   "${prod_controllers_default}" \
-  "${prod_infrastructure_default}"; do
+  "${prod_infrastructure_default}" \
+  "${prod_apps_default}"; do
   assert_default_off "${rendered_path}"
 done
 
@@ -816,6 +936,9 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   fi
 done
 
+assert_heartbeat_staged_after_policy_exclusion "${docker_controllers}" "${docker_apps}"
+assert_heartbeat_staged_after_policy_exclusion "${hetzner_controllers}" "${hetzner_apps}"
+
 assert_auth_proxy_with_coroot "${local_controllers_default}" "${docker_controllers}"
 assert_auth_proxy_with_coroot "${prod_controllers_default}" "${hetzner_controllers}"
 
@@ -824,16 +947,20 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do
   assert_resource_count "${rendered_path}" Coroot observability coroot 0
   assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0
   assert_resource_count "${rendered_path}" HelmRelease monitoring kube-prometheus-stack 1
-  assert_resource_count "${rendered_path}" Secret observability cluster-heartbeat 1
   assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1
-  assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 1
+  assert_resource_count "${rendered_path}" CronJob observability cluster-heartbeat 0
+  assert_resource_count "${rendered_path}" Secret observability cluster-heartbeat 0
+  assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 0
   assert_opencost_absent "${rendered_path}"
   assert_coroot_sso_controller_contract "${rendered_path}"
-  assert_coroot_heartbeat_contract "${rendered_path}"
-  assert_coroot_heartbeat_substitution "${rendered_path}"
   assert_watchdog_enabled "${rendered_path}"
 done
 
+assert_coroot_heartbeat_contract "${docker_apps}" "${docker_controllers}"
+assert_coroot_heartbeat_substitution "${docker_apps}"
+assert_coroot_heartbeat_contract "${hetzner_apps}" "${hetzner_controllers}"
+assert_coroot_heartbeat_substitution "${hetzner_apps}"
+
 for rendered_path in "${docker_infrastructure}" "${hetzner_infrastructure}"; do
   assert_namespace_excluded_from_all_policy_rules "${rendered_path}" add-security-context observability
   assert_namespace_excluded_from_all_policy_rules "${rendered_path}" validate-host-restrictions observability
@@ -999,7 +1126,9 @@ for documented_boundary in \
   "uses a separate optional \`CLUSTER_HEARTBEAT_URL\` secret" \
   "Never point both inputs at the same monitor" \
   "permits only \`hc-ping.com:443\`" \
-  "bootstrap rejects any other configured host" \
+  "Bootstrap accepts only a canonical" \
+  "compares check identities" \
+  "dependent apps layer stages the heartbeat only after" \
   "Secret-backed environment variable" \
   "platform-critical priority" \
   "keep Watchdog unchanged" \
@@ -1037,9 +1166,11 @@ for documented_heartbeat_boundary in \
   "Watchdog stays enabled" \
   "cluster_heartbeat_url" \
   "The three secret inputs" \
-  "only \`hc-ping.com:443\`" \
-  "bootstrap rejects another configured host" \
-  "Keeping Watchdog active also preserves" \
+  "\`hc-ping.com:443\`" \
+  "Bootstrap compares" \
+  "canonical check identities" \
+  "dependent apps layer" \
+  "while Watchdog preserves" \
   "Flux reconciliation alerting still depends on kube-prometheus-stack"; do
   if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then
     echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2
@@ -1050,9 +1181,13 @@ done
 bootstrap_workflow="${repo_root}/.github/workflows/bootstrap.yaml"
 for bootstrap_boundary in \
   "CLUSTER_HEARTBEAT_URL: \${{ secrets.CLUSTER_HEARTBEAT_URL }}" \
-  '""|https://hc-ping.com/*) ;;' \
-  'CLUSTER_HEARTBEAT_URL must use https://hc-ping.com/ or be unset' \
-  "[ \"\${CLUSTER_HEARTBEAT_URL}\" = \"\${ALERTMANAGER_HEARTBEAT_URL}\" ]; then" \
+  "heartbeat_url_pattern='^https://hc-ping\\.com/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$'" \
+  "alertmanager_heartbeat_url_lower=\"\$(printf '%s' \"\${ALERTMANAGER_HEARTBEAT_URL}\" | tr '[:upper:]' '[:lower:]')\"" \
+  'CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/<lowercase-uuid> URL or be unset' \
+  'ALERTMANAGER_HEARTBEAT_URL must be canonical when it uses hc-ping.com' \
+  "cluster_heartbeat_check_id=\"\${BASH_REMATCH[1]}\"" \
+  "alertmanager_heartbeat_check_id=\"\${BASH_REMATCH[1]}\"" \
+  "[ \"\${cluster_heartbeat_check_id}\" = \"\${alertmanager_heartbeat_check_id}\" ]; then" \
   'CLUSTER_HEARTBEAT_URL must use a different external check than ALERTMANAGER_HEARTBEAT_URL' \
   '.stringData.cluster_heartbeat_url = strenv(CLUSTER_HEARTBEAT_URL)'; do
   if ! grep -Fq "${bootstrap_boundary}" "${bootstrap_workflow}"; then
@@ -1061,7 +1196,7 @@ for bootstrap_boundary in \
   fi
 done
 
-if grep -Fq 'alertmanager_heartbeat_url' "${repo_root}/k8s/bases/infrastructure/controllers/coroot/cron-job.yaml"; then
+if grep -Fq 'alertmanager_heartbeat_url' "${repo_root}/k8s/bases/apps/cluster-heartbeat/cron-job.yaml"; then
   echo "Coroot CronJob must not share the Alertmanager pipeline monitor" >&2
   exit 1
 fi
@@ -1069,6 +1204,7 @@ fi
 if grep -R -E -n '(devantler|homelab|hooks\.slack\.com|hcloud|alertmanager_webhook_url)' \
   "${repo_root}/k8s/bases/infrastructure/controllers/coroot" \
   "${repo_root}/k8s/bases/infrastructure/coroot" \
+  "${repo_root}/k8s/bases/apps/cluster-heartbeat" \
   "${repo_root}/k8s/bases/infrastructure/audit-log-forwarder"; then
   echo "Coroot profile manifests contain an instance-specific value" >&2
   exit 1

From cc5aea15bc5b1a14db937541e46fc79f48035f8c Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 06:02:14 +0200
Subject: [PATCH 25/26] fix(observability): reject credentialed heartbeat
 aliases

---
 .github/workflows/bootstrap.yaml         | 3 ++-
 scripts/validate-observability-option.sh | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml
index ea3edc6..0c47e30 100644
--- a/.github/workflows/bootstrap.yaml
+++ b/.github/workflows/bootstrap.yaml
@@ -201,6 +201,7 @@ jobs:
           # than encrypting a silently dead monitor. Empty stays valid so default
           # profiles and operators who have not opted into Coroot remain inert.
           heartbeat_url_pattern='^https://hc-ping\.com/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$'
+          heartbeat_host_pattern='^https?://([^/?#]*@)?hc-ping\.com([:/?#]|$)'
           cluster_heartbeat_check_id=""
           alertmanager_heartbeat_check_id=""
           alertmanager_heartbeat_url_lower="$(printf '%s' "${ALERTMANAGER_HEARTBEAT_URL}" | tr '[:upper:]' '[:lower:]')"
@@ -211,7 +212,7 @@ jobs:
             fi
             cluster_heartbeat_check_id="${BASH_REMATCH[1]}"
           fi
-          if [[ "${alertmanager_heartbeat_url_lower}" == *://hc-ping.com* ]]; then
+          if [[ "${alertmanager_heartbeat_url_lower}" =~ ${heartbeat_host_pattern} ]]; then
             if [[ ! "${ALERTMANAGER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then
               echo "::error::ALERTMANAGER_HEARTBEAT_URL must be canonical when it uses hc-ping.com"
               exit 1
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index 2c3fafd..e50d262 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -209,6 +209,7 @@ assert_bootstrap_heartbeat_identity_contract() {
     "${cluster_url}/1"
     'https://HC-PING.com/11111111-1111-4111-8111-111111111111'
     'https://hc-ping.com:443/11111111-1111-4111-8111-111111111111'
+    'https://user@hc-ping.com/11111111-1111-4111-8111-111111111111'
     'https://hc-ping.com/project-slug/check-slug'
   )
 
@@ -1182,6 +1183,7 @@ bootstrap_workflow="${repo_root}/.github/workflows/bootstrap.yaml"
 for bootstrap_boundary in \
   "CLUSTER_HEARTBEAT_URL: \${{ secrets.CLUSTER_HEARTBEAT_URL }}" \
   "heartbeat_url_pattern='^https://hc-ping\\.com/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$'" \
+  "heartbeat_host_pattern='^https?://([^/?#]*@)?hc-ping\\.com([:/?#]|$)'" \
   "alertmanager_heartbeat_url_lower=\"\$(printf '%s' \"\${ALERTMANAGER_HEARTBEAT_URL}\" | tr '[:upper:]' '[:lower:]')\"" \
   'CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/<lowercase-uuid> URL or be unset' \
   'ALERTMANAGER_HEARTBEAT_URL must be canonical when it uses hc-ping.com' \

From ab1ba55ae62218c463f29194079e9448a4694ac8 Mon Sep 17 00:00:00 2001
From: Nikolai Emil Damm <nikolaiemildamm@icloud.com>
Date: Wed, 22 Jul 2026 06:17:25 +0200
Subject: [PATCH 26/26] fix(observability): keep heartbeat token out of argv

---
 docs/TEMPLATING.md                             |  4 +++-
 docs/dr/alerting.md                            |  4 ++++
 k8s/bases/apps/cluster-heartbeat/cron-job.yaml | 14 ++++++++------
 scripts/validate-observability-option.sh       |  6 ++++--
 4 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md
index abe6463..6a5afc9 100644
--- a/docs/TEMPLATING.md
+++ b/docs/TEMPLATING.md
@@ -98,7 +98,9 @@ plus that hostname's exact DNS query. Bootstrap accepts only a canonical
 fragments, query strings, or trailing slashes cannot make both inputs target the
 same external check under different spellings.
 The substituted bearer URL lives in a Kubernetes Secret and reaches the container
-through a Secret-backed environment variable, not the PodSpec command or arguments.
+through a Secret-backed environment variable. A shell builtin feeds it to curl as
+stdin configuration, so neither the PodSpec nor curl's runtime argument vector
+contains the credential.
 The Coroot infrastructure overlay alone narrows the generated namespace policies
 around the CronJob's purpose-specific label, so default profiles retain their
 global selectors. The dependent apps layer stages the heartbeat only after that
diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md
index 7558273..2a52c98 100644
--- a/docs/dr/alerting.md
+++ b/docs/dr/alerting.md
@@ -72,6 +72,10 @@ workload; default profiles keep their global selectors. The dependent apps layer
 stages the CronJob only after that exclusion is ready, while Watchdog preserves the
 original dead-man path until the apps reconciliation completes. An invalid fallback
 keeps an unconfigured CronJob quiet.
+
+The bearer URL reaches the CronJob through a Secret-backed environment variable.
+The shell feeds it to curl as stdin configuration, so the credential is absent
+from both the PodSpec and curl's runtime argument vector.
 Flux reconciliation alerting still depends on kube-prometheus-stack, so that stack
 remains transitional until a later slice replaces those rules before removing it.
 
diff --git a/k8s/bases/apps/cluster-heartbeat/cron-job.yaml b/k8s/bases/apps/cluster-heartbeat/cron-job.yaml
index a777dd8..161a1ef 100644
--- a/k8s/bases/apps/cluster-heartbeat/cron-job.yaml
+++ b/k8s/bases/apps/cluster-heartbeat/cron-job.yaml
@@ -8,10 +8,11 @@
 # cluster-down alarm.
 #
 # The URL comes from the dedicated Flux substitution input through a Secret
-# reference, so the bearer token does not appear in the PodSpec or argv. Local
-# and unconfigured clusters use the invalid fallback, and `|| true` keeps the
-# Job from flapping when no external monitor is configured. The signal is the
-# absence of pings at the external monitor, not the Job exit code.
+# reference, and curl consumes it from stdin configuration, so the bearer token
+# does not appear in the PodSpec or curl's runtime argv. Local and unconfigured
+# clusters use the invalid fallback, and `|| true` keeps the Job from flapping
+# when no external monitor is configured. The signal is the absence of pings at
+# the external monitor, not the Job exit code.
 apiVersion: batch/v1
 kind: CronJob
 metadata:
@@ -49,9 +50,10 @@ spec:
                 - -c
               args:
                 - >-
-                  curl -sf --max-time 10
+                  printf 'url = "%s"\n' "$HEARTBEAT_URL" |
+                  curl --config - -sf --max-time 10
                   --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused
-                  "$HEARTBEAT_URL" || true
+                  || true
               env:
                 - name: HEARTBEAT_URL
                   valueFrom:
diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh
index e50d262..fbf6957 100755
--- a/scripts/validate-observability-option.sh
+++ b/scripts/validate-observability-option.sh
@@ -276,7 +276,7 @@ assert_coroot_heartbeat_contract() {
           "-c"
         ]) |
         select($container.args == [
-          "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"$HEARTBEAT_URL\" || true"
+          "printf '\''url = \"%s\"\\n'\'' \"$HEARTBEAT_URL\" | curl --config - -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused || true"
         ]) |
         select($container.env == [{
           "name":"HEARTBEAT_URL",
@@ -398,7 +398,7 @@ assert_coroot_heartbeat_substitution() {
         .spec.jobTemplate.spec.template.spec.containers[0] as $container |
         select($container.command == ["/bin/sh", "-c"]) |
         select($container.args == [
-          "curl -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused \"$HEARTBEAT_URL\" || true"
+          "printf '\''url = \"%s\"\\n'\'' \"$HEARTBEAT_URL\" | curl --config - -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused || true"
         ]) |
         select($container.env == [{
           "name":"HEARTBEAT_URL",
@@ -1131,6 +1131,7 @@ for documented_boundary in \
   "compares check identities" \
   "dependent apps layer stages the heartbeat only after" \
   "Secret-backed environment variable" \
+  "stdin configuration" \
   "platform-critical priority" \
   "keep Watchdog unchanged" \
   "preserves a dead-man signal" \
@@ -1172,6 +1173,7 @@ for documented_heartbeat_boundary in \
   "canonical check identities" \
   "dependent apps layer" \
   "while Watchdog preserves" \
+  "curl as stdin configuration" \
   "Flux reconciliation alerting still depends on kube-prometheus-stack"; do
   if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then
     echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2