feat(observability): stage Coroot cluster heartbeat

.github/workflows/bootstrap.yaml

@@ -179,12 +179,14 @@ jobs:
           grep -rIl 'example-org' k8s/ 2>/dev/null | xargs -r sed -i "s|example-org|${OWNER}|g"
 
       - name: 🔐 Encrypt secrets into *.enc.yaml
+        shell: bash
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
           R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
           R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
           ALERTMANAGER_WEBHOOK_URL: ${{ secrets.ALERTMANAGER_WEBHOOK_URL }}
           ALERTMANAGER_HEARTBEAT_URL: ${{ secrets.ALERTMANAGER_HEARTBEAT_URL }}
+          CLUSTER_HEARTBEAT_URL: ${{ secrets.CLUSTER_HEARTBEAT_URL }}
           GITHUB_APP_CLIENT_SECRET: ${{ secrets.GITHUB_APP_CLIENT_SECRET }}
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
         run: |
@@ -194,6 +196,35 @@ jobs:
           export FLUX_WEB_CLIENT_SECRET="$(openssl rand -hex 32)"
           export OAUTH2_PROXY_COOKIE_SECRET="$(openssl rand -base64 32)"
 
+          # The dedicated Coroot heartbeat has an exact hc-ping.com egress
+          # policy. Reject a configured URL that the cluster cannot reach rather
+          # than encrypting a silently dead monitor. Empty stays valid so default
+          # profiles and operators who have not opted into Coroot remain inert.
+          heartbeat_url_pattern='^https://hc-ping\.com/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$'
+          heartbeat_host_pattern='^https?://([^/?#]*@)?hc-ping\.com([:/?#]|$)'
+          cluster_heartbeat_check_id=""
+          alertmanager_heartbeat_check_id=""
+          alertmanager_heartbeat_url_lower="$(printf '%s' "${ALERTMANAGER_HEARTBEAT_URL}" | tr '[:upper:]' '[:lower:]')"
+          if [ -n "${CLUSTER_HEARTBEAT_URL}" ]; then
+            if [[ ! "${CLUSTER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then
+              echo "::error::CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/<lowercase-uuid> URL or be unset"
+              exit 1
+            fi
+            cluster_heartbeat_check_id="${BASH_REMATCH[1]}"
+          fi
+          if [[ "${alertmanager_heartbeat_url_lower}" =~ ${heartbeat_host_pattern} ]]; then
+            if [[ ! "${ALERTMANAGER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then
+              echo "::error::ALERTMANAGER_HEARTBEAT_URL must be canonical when it uses hc-ping.com"
+              exit 1
+            fi
+            alertmanager_heartbeat_check_id="${BASH_REMATCH[1]}"
+          fi
+          if [ -n "${cluster_heartbeat_check_id}" ] && \
+             [ "${cluster_heartbeat_check_id}" = "${alertmanager_heartbeat_check_id}" ]; then
+            echo "::error::CLUSTER_HEARTBEAT_URL must use a different external check than ALERTMANAGER_HEARTBEAT_URL"
+            exit 1
+          fi
+
           fill_base() { # variables-base-secret
             yq -i '
               .stringData.cloudflare_api_token = strenv(CLOUDFLARE_API_TOKEN) |
@@ -207,6 +238,7 @@ jobs:
           yq -i '
             .stringData.alertmanager_webhook_url = strenv(ALERTMANAGER_WEBHOOK_URL) |
             .stringData.alertmanager_heartbeat_url = strenv(ALERTMANAGER_HEARTBEAT_URL) |
+            .stringData.cluster_heartbeat_url = strenv(CLUSTER_HEARTBEAT_URL) |
             .stringData.dex_client_secret = strenv(DEX_CLIENT_SECRET) |
             .stringData.flux_web_client_secret = strenv(FLUX_WEB_CLIENT_SECRET) |
             .stringData.oauth2_proxy_cookie_secret = strenv(OAUTH2_PROXY_COOKIE_SECRET) |

docs/BOOTSTRAP.md

@@ -111,7 +111,8 @@ Set these in your **new instance repository** under *Settings → Secrets and va
 | `R2_ACCESS_KEY_ID` | R2 (S3) access key ID for backups | Cloudflare R2 → **Manage R2 API Tokens**. |
 | `R2_SECRET_ACCESS_KEY` | R2 (S3) secret access key for backups | Same place as the access key ID. |
 | `ALERTMANAGER_WEBHOOK_URL` | Incoming webhook for alert notifications (e.g. Slack) | Your chat platform’s incoming-webhook setup. |
-| `ALERTMANAGER_HEARTBEAT_URL` | Ping URL for the off-cluster dead-man’s-switch | healthchecks.io (or similar) check ping URL. |
+| `ALERTMANAGER_HEARTBEAT_URL` | Ping URL that proves the Prometheus → Alertmanager pipeline is alive | healthchecks.io (or similar) check ping URL. |
+| `CLUSTER_HEARTBEAT_URL` | Optional Coroot-profile cluster-liveness ping URL | A **separate**, canonical `https://hc-ping.com/<lowercase-uuid>` URL; never reuse the Alertmanager check under another URL spelling. |
 | `GITHUB_APP_CLIENT_SECRET` | **Client secret** of the *cluster SSO* GitHub App (Dex’s GitHub connector) | The SSO App’s settings page (generate a client secret). |
 | `APP_PRIVATE_KEY` | Private key (`.pem`) of the **bootstrap** GitHub App | Generated when you created the bootstrap App. |
 

docs/TEMPLATING.md

@@ -88,8 +88,30 @@ variable for Coroot incident and resolution notifications. Its matching provider
 policy permits only `hooks.slack.com:443`, so keep this value a Slack incoming-webhook URL.
 Per-alert notifications remain visible only in the Coroot UI; this avoids sending
 every noisy alert to the shared webhook. Local / Docker Coroot stays notification-free.
-Kube-prometheus-stack keeps owning its alert rules until that remaining metrics
-and alerting migration is delivered separately.
+The profile also stages one hardened `cluster-heartbeat` CronJob for the cluster
+dead-man signal. It uses a separate optional `CLUSTER_HEARTBEAT_URL` secret so
+the existing Alertmanager Watchdog check keeps proving the Prometheus-to-Alertmanager
+pipeline. Never point both inputs at the same monitor. The CronJob runs at
+platform-critical priority, and its dedicated policy permits only `hc-ping.com:443`
+plus that hostname's exact DNS query. Bootstrap accepts only a canonical
+`https://hc-ping.com/<lowercase-uuid>` URL and compares check identities, so
+fragments, query strings, or trailing slashes cannot make both inputs target the
+same external check under different spellings.
+The substituted bearer URL lives in a Kubernetes Secret and reaches the container
+through a Secret-backed environment variable. A shell builtin feeds it to curl as
+stdin configuration, so neither the PodSpec nor curl's runtime argument vector
+contains the credential.
+The Coroot infrastructure overlay alone narrows the generated namespace policies
+around the CronJob's purpose-specific label, so default profiles retain their
+global selectors. The dependent apps layer stages the heartbeat only after that
+infrastructure reconciliation is ready, preventing an existing namespace-wide
+deny policy from blocking the new workload during an upgrade. The invalid URL
+fallback keeps local and unconfigured instances inert. Both default and Coroot
+profiles keep Watchdog unchanged on the distinct Alertmanager pipeline monitor,
+which preserves a dead-man signal until the apps layer starts the direct check.
+Kube-prometheus-stack keeps owning its remaining alert
+rules, including Flux reconciliation alerts, until that migration is delivered
+separately.
 
 Open the Coroot UI at `https://observability.<your-domain>`. Community Edition
 does not provide native OIDC, so this profile sends every UI request through the

docs/dr/alerting.md

@@ -6,9 +6,9 @@ shipper) and **OpenCost** (cost), running per cluster. Self-hosted, no
 SaaS metrics tier. The stack is production-hardened along four axes:
 
 1. **Alerts go to Slack** via Alertmanager's native `slack_configs`.
-2. **A dead-man's-switch** (the always-firing `Watchdog` alert) is pushed
-   to an external heartbeat monitor — the one failure mode in-cluster
-   alerting can never cover (the whole cluster being down).
+2. **Two independent dead-man checks** separate the Prometheus → Alertmanager
+   pipeline from direct cluster liveness, so one sender cannot hide the other
+   path's failure.
 3. **State is persistent** — Prometheus, Alertmanager and Loki use Hetzner
    Cloud block volumes in prod, and Velero ships the `monitoring`
    namespace to R2 daily (24 h RPO).
@@ -51,22 +51,47 @@ clusters get an invalid URL and stay quiet by design.
 ## Dead-man's-switch (off-cluster heartbeat)
 
 In-cluster Alertmanager cannot tell you the cluster is down — it's down
-too. To cover that, the chart's always-firing `Watchdog` alert is routed
-to a dedicated `heartbeat` receiver that POSTs to an **external** monitor
-on a tight cadence (`repeat_interval: 50s`). If the cluster — or the
-Prometheus → Alertmanager pipeline — dies, the monitor stops receiving
-pings and notifies Slack out-of-band.
+too. The default profile keeps the `Watchdog` alert and routes it to a
+dedicated `heartbeat` receiver that POSTs to an **external** monitor on a
+tight cadence (`repeat_interval: 50s`). If the cluster — or the Prometheus →
+Alertmanager pipeline — dies, the monitor stops receiving pings and notifies
+Slack out-of-band.
+
+The Coroot profile adds a dedicated `cluster-heartbeat` CronJob, which pings a
+**second external check** every five minutes independently of Coroot. It never
+reuses `alertmanager_heartbeat_url`: doing so would let the CronJob keep that
+monitor green while the Prometheus-to-Alertmanager pipeline was down.
+Watchdog stays enabled on its existing check in both default and Coroot profiles.
+The CronJob's separate `cluster_heartbeat_url` accepts only a canonical
+`https://hc-ping.com/<lowercase-uuid>` URL; its network policy allows only
+`hc-ping.com:443` plus that hostname's exact DNS query. Bootstrap compares
+canonical check identities, so URL fragments, query strings, and trailing slashes
+cannot disguise reuse of the Alertmanager check. The Coroot infrastructure overlay
+alone narrows the namespace deny and DNS policies around this purpose-labelled
+workload; default profiles keep their global selectors. The dependent apps layer
+stages the CronJob only after that exclusion is ready, while Watchdog preserves the
+original dead-man path until the apps reconciliation completes. An invalid fallback
+keeps an unconfigured CronJob quiet.
+
+The bearer URL reaches the CronJob through a Secret-backed environment variable.
+The shell feeds it to curl as stdin configuration, so the credential is absent
+from both the PodSpec and curl's runtime argument vector.
+Flux reconciliation alerting still depends on kube-prometheus-stack, so that stack
+remains transitional until a later slice replaces those rules before removing it.
 
 Recommended monitor: [healthchecks.io](https://healthchecks.io) (free,
-open-source, native Slack integration). Create a check with a ~5 min
-period and ~10 min grace, connect it to Slack, and put its ping URL in
-`alertmanager_heartbeat_url` (below). A self-hosted alternative is a
+open-source, native Slack integration). Keep two distinct checks: the existing
+tight-cadence Watchdog URL in `alertmanager_heartbeat_url`, and a ~5 min period /
+~10 min grace direct-cluster check in `cluster_heartbeat_url`. Connect both to
+Slack, but never give them the same ping URL. A self-hosted alternative for the
+Alertmanager check is supported; the CronJob deliberately accepts only
+`hc-ping.com` because its egress policy is fail closed. Another alternative is a
 scheduled GitHub Actions workflow that probes the public Gateway and posts
 to Slack — fully under your control, no third-party monitor.
 
-The heartbeat URL is injected by Flux substitution
-(`${alertmanager_heartbeat_url}`); unset, it defaults to an invalid URL,
-so local/CI simply never heartbeat — harmless.
+Both URLs are injected by Flux substitution. An unset `cluster_heartbeat_url`
+defaults to an invalid URL, so local/CI and Coroot profiles without the optional
+second check stay inert while Watchdog continues unchanged.
 
 ## Off-cluster metric/log backup
 
@@ -119,30 +144,36 @@ Two sources:
 
 ## Per-environment setup (manual SOPS steps)
 
-The Slack webhook and heartbeat URL are secrets, so they live in the
-per-cluster `variables-cluster-secret.enc.yaml` (under `bootstrap/`) and
-must be set by hand. Both are read from the `Secret` `variables-cluster`,
-which is a Flux `substituteFrom` source.
+The three secret inputs — `alertmanager_webhook_url`,
+`alertmanager_heartbeat_url`, and `cluster_heartbeat_url` — live in the
+per-cluster `variables-cluster-secret.enc.yaml` (under `bootstrap/`) and must
+be set by hand. All three are read from the `Secret` `variables-cluster`, which
+is a Flux `substituteFrom` source.
 
 ```bash
 # 1. Slack incoming webhook for alert notifications.
 sops --set '["stringData"]["alertmanager_webhook_url"] "https://hooks.slack.com/services/XXX/YYY/ZZZ"' \
   k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
 
-# 2. External heartbeat-monitor ping URL (e.g. healthchecks.io).
+# 2. Prometheus → Alertmanager pipeline check (healthchecks.io or similar).
 sops --set '["stringData"]["alertmanager_heartbeat_url"] "https://hc-ping.com/<uuid>"' \
   k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
+
+# 3. Separate direct-cluster check for the Coroot profile (hc-ping.com only).
+sops --set '["stringData"]["cluster_heartbeat_url"] "https://hc-ping.com/<different-uuid>"' \
+  k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml
 ```
 
 Slack side: create an incoming webhook for your alerts channel (e.g.
 `#platform-alerts`) — the channel in the config is cosmetic, since an incoming
-webhook posts to the channel it was created for. healthchecks.io side: create
-the check, connect its Slack integration, copy the ping URL.
-
-| Env   | `alertmanager_webhook_url`        | `alertmanager_heartbeat_url`       |
-| ----- | --------------------------------- | ---------------------------------- |
-| local | invalid URL (alerts stay local)   | unset → invalid (no heartbeat)     |
-| prod  | Slack alerts-channel webhook      | healthchecks.io ping URL           |
+webhook posts to the channel it was created for. On healthchecks.io, create two
+distinct checks, connect each check's Slack integration, and copy their different
+ping URLs into `alertmanager_heartbeat_url` and `cluster_heartbeat_url` respectively.
+
+| Env   | `alertmanager_webhook_url`      | `alertmanager_heartbeat_url`       | `cluster_heartbeat_url`                 |
+| ----- | ------------------------------- | ---------------------------------- | --------------------------------------- |
+| local | invalid URL (alerts stay local) | unset → invalid                    | unset → invalid                         |
+| prod  | Slack alerts-channel webhook    | pipeline-monitor ping URL          | distinct Coroot cluster-monitor ping URL |
 
 ## On-call: silence and inspect
 

k8s/bases/apps/cluster-heartbeat/cilium-network-policy.yaml

@@ -0,0 +1,35 @@
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: allow-cluster-heartbeat
+  namespace: observability
+spec:
+  endpointSelector:
+    matchLabels:
+      platform-heartbeat: "true"
+  enableDefaultDeny:
+    ingress: true
+    egress: true
+  ingressDeny:
+    - {}
+  egress:
+    - toFQDNs:
+        - matchName: hc-ping.com
+      toPorts:
+        - ports:
+            - port: "443"
+              protocol: TCP
+    # DNS is the only other egress path required to resolve the exact endpoint.
+    - toEndpoints:
+        - matchLabels:
+            k8s:io.kubernetes.pod.namespace: kube-system
+            k8s-app: kube-dns
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+            - port: "53"
+              protocol: TCP
+          rules:
+            dns:
+              - matchName: hc-ping.com

k8s/bases/apps/cluster-heartbeat/cron-job.yaml

@@ -0,0 +1,78 @@
+# Cluster-level dead-man's-switch. Every five minutes a pod pings the
+# per-cluster external heartbeat monitor. The monitor alerts out of band when
+# pings stop, covering the one failure that in-cluster alerting cannot report:
+# the whole cluster being unavailable.
+#
+# The ping is unconditional. Component health remains Coroot's responsibility;
+# coupling this signal to Coroot would turn a component outage into a false
+# cluster-down alarm.
+#
+# The URL comes from the dedicated Flux substitution input through a Secret
+# reference, and curl consumes it from stdin configuration, so the bearer token
+# does not appear in the PodSpec or curl's runtime argv. Local and unconfigured
+# clusters use the invalid fallback, and `|| true` keeps the Job from flapping
+# when no external monitor is configured. The signal is the absence of pings at
+# the external monitor, not the Job exit code.
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cluster-heartbeat
+  namespace: observability
+spec:
+  schedule: "*/5 * * * *"
+  concurrencyPolicy: Forbid
+  startingDeadlineSeconds: 30
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 120
+      template:
+        metadata:
+          labels:
+            app: cluster-heartbeat
+            platform-heartbeat: "true"
+        spec:
+          restartPolicy: Never
+          automountServiceAccountToken: false
+          priorityClassName: platform-critical
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: heartbeat
+              image: docker.io/curlimages/curl:8.21.0@sha256:7c12af72ceb38b7432ab85e1a265cff6ae58e06f95539d539b654f2cfa64bb13
+              command:
+                - /bin/sh
+                - -c
+              args:
+                - >-
+                  printf 'url = "%s"\n' "$HEARTBEAT_URL" |
+                  curl --config - -sf --max-time 10
+                  --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused
+                  || true
+              env:
+                - name: HEARTBEAT_URL
+                  valueFrom:
+                    secretKeyRef:
+                      name: cluster-heartbeat
+                      key: url
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                runAsNonRoot: true
+                runAsUser: 65532
+                runAsGroup: 65532
+                capabilities:
+                  drop:
+                    - ALL
+              resources:
+                requests:
+                  cpu: 5m
+                  memory: 16Mi
+                limits:
+                  cpu: 50m
+                  memory: 32Mi

k8s/bases/apps/cluster-heartbeat/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# The apps Flux layer depends on infrastructure. Keeping the heartbeat here
+# ensures the scoped observability deny-policy replacement is ready before the
+# purpose-labelled workload starts during an existing-cluster upgrade.
+resources:
+  - cilium-network-policy.yaml
+  - cron-job.yaml
+  - secret.yaml

k8s/bases/apps/cluster-heartbeat/secret.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cluster-heartbeat
+  namespace: observability
+type: Opaque
+stringData:
+  # Flux substitutes the encrypted cluster variable here. The CronJob reads the
+  # resulting Secret so the bearer URL never appears in its PodSpec or argv.
+  url: "${cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}"

k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml

@@ -4,7 +4,12 @@ metadata:
   name: allow-coroot
   namespace: observability
 spec:
-  endpointSelector: {}
+  endpointSelector:
+    matchExpressions:
+      - key: platform-heartbeat
+        operator: NotIn
+        values:
+          - "true"
   ingress:
     # Coroot, its bundled stores, its agents, and the audit forwarder exchange
     # telemetry inside this namespace.