diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/bootstrap.yaml index 094a1e2..0c47e30 100644 --- a/.github/workflows/bootstrap.yaml +++ b/.github/workflows/bootstrap.yaml @@ -179,12 +179,14 @@ jobs: grep -rIl 'example-org' k8s/ 2>/dev/null | xargs -r sed -i "s|example-org|${OWNER}|g" - name: πŸ” Encrypt secrets into *.enc.yaml + shell: bash env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} ALERTMANAGER_WEBHOOK_URL: ${{ secrets.ALERTMANAGER_WEBHOOK_URL }} ALERTMANAGER_HEARTBEAT_URL: ${{ secrets.ALERTMANAGER_HEARTBEAT_URL }} + CLUSTER_HEARTBEAT_URL: ${{ secrets.CLUSTER_HEARTBEAT_URL }} GITHUB_APP_CLIENT_SECRET: ${{ secrets.GITHUB_APP_CLIENT_SECRET }} HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} run: | @@ -194,6 +196,35 @@ jobs: export FLUX_WEB_CLIENT_SECRET="$(openssl rand -hex 32)" export OAUTH2_PROXY_COOKIE_SECRET="$(openssl rand -base64 32)" + # The dedicated Coroot heartbeat has an exact hc-ping.com egress + # policy. Reject a configured URL that the cluster cannot reach rather + # than encrypting a silently dead monitor. Empty stays valid so default + # profiles and operators who have not opted into Coroot remain inert. + heartbeat_url_pattern='^https://hc-ping\.com/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$' + heartbeat_host_pattern='^https?://([^/?#]*@)?hc-ping\.com([:/?#]|$)' + cluster_heartbeat_check_id="" + alertmanager_heartbeat_check_id="" + alertmanager_heartbeat_url_lower="$(printf '%s' "${ALERTMANAGER_HEARTBEAT_URL}" | tr '[:upper:]' '[:lower:]')" + if [ -n "${CLUSTER_HEARTBEAT_URL}" ]; then + if [[ ! "${CLUSTER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then + echo "::error::CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/ URL or be unset" + exit 1 + fi + cluster_heartbeat_check_id="${BASH_REMATCH[1]}" + fi + if [[ "${alertmanager_heartbeat_url_lower}" =~ ${heartbeat_host_pattern} ]]; then + if [[ ! "${ALERTMANAGER_HEARTBEAT_URL}" =~ ${heartbeat_url_pattern} ]]; then + echo "::error::ALERTMANAGER_HEARTBEAT_URL must be canonical when it uses hc-ping.com" + exit 1 + fi + alertmanager_heartbeat_check_id="${BASH_REMATCH[1]}" + fi + if [ -n "${cluster_heartbeat_check_id}" ] && \ + [ "${cluster_heartbeat_check_id}" = "${alertmanager_heartbeat_check_id}" ]; then + echo "::error::CLUSTER_HEARTBEAT_URL must use a different external check than ALERTMANAGER_HEARTBEAT_URL" + exit 1 + fi + fill_base() { # variables-base-secret yq -i ' .stringData.cloudflare_api_token = strenv(CLOUDFLARE_API_TOKEN) | @@ -207,6 +238,7 @@ jobs: yq -i ' .stringData.alertmanager_webhook_url = strenv(ALERTMANAGER_WEBHOOK_URL) | .stringData.alertmanager_heartbeat_url = strenv(ALERTMANAGER_HEARTBEAT_URL) | + .stringData.cluster_heartbeat_url = strenv(CLUSTER_HEARTBEAT_URL) | .stringData.dex_client_secret = strenv(DEX_CLIENT_SECRET) | .stringData.flux_web_client_secret = strenv(FLUX_WEB_CLIENT_SECRET) | .stringData.oauth2_proxy_cookie_secret = strenv(OAUTH2_PROXY_COOKIE_SECRET) | diff --git a/docs/BOOTSTRAP.md b/docs/BOOTSTRAP.md index f868851..d91e4d4 100644 --- a/docs/BOOTSTRAP.md +++ b/docs/BOOTSTRAP.md @@ -111,7 +111,8 @@ Set these in your **new instance repository** under *Settings β†’ Secrets and va | `R2_ACCESS_KEY_ID` | R2 (S3) access key ID for backups | Cloudflare R2 β†’ **Manage R2 API Tokens**. | | `R2_SECRET_ACCESS_KEY` | R2 (S3) secret access key for backups | Same place as the access key ID. | | `ALERTMANAGER_WEBHOOK_URL` | Incoming webhook for alert notifications (e.g. Slack) | Your chat platform’s incoming-webhook setup. | -| `ALERTMANAGER_HEARTBEAT_URL` | Ping URL for the off-cluster dead-man’s-switch | healthchecks.io (or similar) check ping URL. | +| `ALERTMANAGER_HEARTBEAT_URL` | Ping URL that proves the Prometheus β†’ Alertmanager pipeline is alive | healthchecks.io (or similar) check ping URL. | +| `CLUSTER_HEARTBEAT_URL` | Optional Coroot-profile cluster-liveness ping URL | A **separate**, canonical `https://hc-ping.com/` URL; never reuse the Alertmanager check under another URL spelling. | | `GITHUB_APP_CLIENT_SECRET` | **Client secret** of the *cluster SSO* GitHub App (Dex’s GitHub connector) | The SSO App’s settings page (generate a client secret). | | `APP_PRIVATE_KEY` | Private key (`.pem`) of the **bootstrap** GitHub App | Generated when you created the bootstrap App. | diff --git a/docs/TEMPLATING.md b/docs/TEMPLATING.md index 500dc85..6a5afc9 100644 --- a/docs/TEMPLATING.md +++ b/docs/TEMPLATING.md @@ -88,8 +88,30 @@ variable for Coroot incident and resolution notifications. Its matching provider policy permits only `hooks.slack.com:443`, so keep this value a Slack incoming-webhook URL. Per-alert notifications remain visible only in the Coroot UI; this avoids sending every noisy alert to the shared webhook. Local / Docker Coroot stays notification-free. -Kube-prometheus-stack keeps owning its alert rules until that remaining metrics -and alerting migration is delivered separately. +The profile also stages one hardened `cluster-heartbeat` CronJob for the cluster +dead-man signal. It uses a separate optional `CLUSTER_HEARTBEAT_URL` secret so +the existing Alertmanager Watchdog check keeps proving the Prometheus-to-Alertmanager +pipeline. Never point both inputs at the same monitor. The CronJob runs at +platform-critical priority, and its dedicated policy permits only `hc-ping.com:443` +plus that hostname's exact DNS query. Bootstrap accepts only a canonical +`https://hc-ping.com/` URL and compares check identities, so +fragments, query strings, or trailing slashes cannot make both inputs target the +same external check under different spellings. +The substituted bearer URL lives in a Kubernetes Secret and reaches the container +through a Secret-backed environment variable. A shell builtin feeds it to curl as +stdin configuration, so neither the PodSpec nor curl's runtime argument vector +contains the credential. +The Coroot infrastructure overlay alone narrows the generated namespace policies +around the CronJob's purpose-specific label, so default profiles retain their +global selectors. The dependent apps layer stages the heartbeat only after that +infrastructure reconciliation is ready, preventing an existing namespace-wide +deny policy from blocking the new workload during an upgrade. The invalid URL +fallback keeps local and unconfigured instances inert. Both default and Coroot +profiles keep Watchdog unchanged on the distinct Alertmanager pipeline monitor, +which preserves a dead-man signal until the apps layer starts the direct check. +Kube-prometheus-stack keeps owning its remaining alert +rules, including Flux reconciliation alerts, until that migration is delivered +separately. Open the Coroot UI at `https://observability.`. Community Edition does not provide native OIDC, so this profile sends every UI request through the diff --git a/docs/dr/alerting.md b/docs/dr/alerting.md index 9ba8401..2a52c98 100644 --- a/docs/dr/alerting.md +++ b/docs/dr/alerting.md @@ -6,9 +6,9 @@ shipper) and **OpenCost** (cost), running per cluster. Self-hosted, no SaaS metrics tier. The stack is production-hardened along four axes: 1. **Alerts go to Slack** via Alertmanager's native `slack_configs`. -2. **A dead-man's-switch** (the always-firing `Watchdog` alert) is pushed - to an external heartbeat monitor β€” the one failure mode in-cluster - alerting can never cover (the whole cluster being down). +2. **Two independent dead-man checks** separate the Prometheus β†’ Alertmanager + pipeline from direct cluster liveness, so one sender cannot hide the other + path's failure. 3. **State is persistent** β€” Prometheus, Alertmanager and Loki use Hetzner Cloud block volumes in prod, and Velero ships the `monitoring` namespace to R2 daily (24 h RPO). @@ -51,22 +51,47 @@ clusters get an invalid URL and stay quiet by design. ## Dead-man's-switch (off-cluster heartbeat) In-cluster Alertmanager cannot tell you the cluster is down β€” it's down -too. To cover that, the chart's always-firing `Watchdog` alert is routed -to a dedicated `heartbeat` receiver that POSTs to an **external** monitor -on a tight cadence (`repeat_interval: 50s`). If the cluster β€” or the -Prometheus β†’ Alertmanager pipeline β€” dies, the monitor stops receiving -pings and notifies Slack out-of-band. +too. The default profile keeps the `Watchdog` alert and routes it to a +dedicated `heartbeat` receiver that POSTs to an **external** monitor on a +tight cadence (`repeat_interval: 50s`). If the cluster β€” or the Prometheus β†’ +Alertmanager pipeline β€” dies, the monitor stops receiving pings and notifies +Slack out-of-band. + +The Coroot profile adds a dedicated `cluster-heartbeat` CronJob, which pings a +**second external check** every five minutes independently of Coroot. It never +reuses `alertmanager_heartbeat_url`: doing so would let the CronJob keep that +monitor green while the Prometheus-to-Alertmanager pipeline was down. +Watchdog stays enabled on its existing check in both default and Coroot profiles. +The CronJob's separate `cluster_heartbeat_url` accepts only a canonical +`https://hc-ping.com/` URL; its network policy allows only +`hc-ping.com:443` plus that hostname's exact DNS query. Bootstrap compares +canonical check identities, so URL fragments, query strings, and trailing slashes +cannot disguise reuse of the Alertmanager check. The Coroot infrastructure overlay +alone narrows the namespace deny and DNS policies around this purpose-labelled +workload; default profiles keep their global selectors. The dependent apps layer +stages the CronJob only after that exclusion is ready, while Watchdog preserves the +original dead-man path until the apps reconciliation completes. An invalid fallback +keeps an unconfigured CronJob quiet. + +The bearer URL reaches the CronJob through a Secret-backed environment variable. +The shell feeds it to curl as stdin configuration, so the credential is absent +from both the PodSpec and curl's runtime argument vector. +Flux reconciliation alerting still depends on kube-prometheus-stack, so that stack +remains transitional until a later slice replaces those rules before removing it. Recommended monitor: [healthchecks.io](https://healthchecks.io) (free, -open-source, native Slack integration). Create a check with a ~5 min -period and ~10 min grace, connect it to Slack, and put its ping URL in -`alertmanager_heartbeat_url` (below). A self-hosted alternative is a +open-source, native Slack integration). Keep two distinct checks: the existing +tight-cadence Watchdog URL in `alertmanager_heartbeat_url`, and a ~5 min period / +~10 min grace direct-cluster check in `cluster_heartbeat_url`. Connect both to +Slack, but never give them the same ping URL. A self-hosted alternative for the +Alertmanager check is supported; the CronJob deliberately accepts only +`hc-ping.com` because its egress policy is fail closed. Another alternative is a scheduled GitHub Actions workflow that probes the public Gateway and posts to Slack β€” fully under your control, no third-party monitor. -The heartbeat URL is injected by Flux substitution -(`${alertmanager_heartbeat_url}`); unset, it defaults to an invalid URL, -so local/CI simply never heartbeat β€” harmless. +Both URLs are injected by Flux substitution. An unset `cluster_heartbeat_url` +defaults to an invalid URL, so local/CI and Coroot profiles without the optional +second check stay inert while Watchdog continues unchanged. ## Off-cluster metric/log backup @@ -119,30 +144,36 @@ Two sources: ## Per-environment setup (manual SOPS steps) -The Slack webhook and heartbeat URL are secrets, so they live in the -per-cluster `variables-cluster-secret.enc.yaml` (under `bootstrap/`) and -must be set by hand. Both are read from the `Secret` `variables-cluster`, -which is a Flux `substituteFrom` source. +The three secret inputs β€” `alertmanager_webhook_url`, +`alertmanager_heartbeat_url`, and `cluster_heartbeat_url` β€” live in the +per-cluster `variables-cluster-secret.enc.yaml` (under `bootstrap/`) and must +be set by hand. All three are read from the `Secret` `variables-cluster`, which +is a Flux `substituteFrom` source. ```bash # 1. Slack incoming webhook for alert notifications. sops --set '["stringData"]["alertmanager_webhook_url"] "https://hooks.slack.com/services/XXX/YYY/ZZZ"' \ k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml -# 2. External heartbeat-monitor ping URL (e.g. healthchecks.io). +# 2. Prometheus β†’ Alertmanager pipeline check (healthchecks.io or similar). sops --set '["stringData"]["alertmanager_heartbeat_url"] "https://hc-ping.com/"' \ k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml + +# 3. Separate direct-cluster check for the Coroot profile (hc-ping.com only). +sops --set '["stringData"]["cluster_heartbeat_url"] "https://hc-ping.com/"' \ + k8s/clusters/prod/bootstrap/variables-cluster-secret.enc.yaml ``` Slack side: create an incoming webhook for your alerts channel (e.g. `#platform-alerts`) β€” the channel in the config is cosmetic, since an incoming -webhook posts to the channel it was created for. healthchecks.io side: create -the check, connect its Slack integration, copy the ping URL. - -| Env | `alertmanager_webhook_url` | `alertmanager_heartbeat_url` | -| ----- | --------------------------------- | ---------------------------------- | -| local | invalid URL (alerts stay local) | unset β†’ invalid (no heartbeat) | -| prod | Slack alerts-channel webhook | healthchecks.io ping URL | +webhook posts to the channel it was created for. On healthchecks.io, create two +distinct checks, connect each check's Slack integration, and copy their different +ping URLs into `alertmanager_heartbeat_url` and `cluster_heartbeat_url` respectively. + +| Env | `alertmanager_webhook_url` | `alertmanager_heartbeat_url` | `cluster_heartbeat_url` | +| ----- | ------------------------------- | ---------------------------------- | --------------------------------------- | +| local | invalid URL (alerts stay local) | unset β†’ invalid | unset β†’ invalid | +| prod | Slack alerts-channel webhook | pipeline-monitor ping URL | distinct Coroot cluster-monitor ping URL | ## On-call: silence and inspect diff --git a/k8s/bases/apps/cluster-heartbeat/cilium-network-policy.yaml b/k8s/bases/apps/cluster-heartbeat/cilium-network-policy.yaml new file mode 100644 index 0000000..a3ae947 --- /dev/null +++ b/k8s/bases/apps/cluster-heartbeat/cilium-network-policy.yaml @@ -0,0 +1,35 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: allow-cluster-heartbeat + namespace: observability +spec: + endpointSelector: + matchLabels: + platform-heartbeat: "true" + enableDefaultDeny: + ingress: true + egress: true + ingressDeny: + - {} + egress: + - toFQDNs: + - matchName: hc-ping.com + toPorts: + - ports: + - port: "443" + protocol: TCP + # DNS is the only other egress path required to resolve the exact endpoint. + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP + rules: + dns: + - matchName: hc-ping.com diff --git a/k8s/bases/apps/cluster-heartbeat/cron-job.yaml b/k8s/bases/apps/cluster-heartbeat/cron-job.yaml new file mode 100644 index 0000000..161a1ef --- /dev/null +++ b/k8s/bases/apps/cluster-heartbeat/cron-job.yaml @@ -0,0 +1,78 @@ +# Cluster-level dead-man's-switch. Every five minutes a pod pings the +# per-cluster external heartbeat monitor. The monitor alerts out of band when +# pings stop, covering the one failure that in-cluster alerting cannot report: +# the whole cluster being unavailable. +# +# The ping is unconditional. Component health remains Coroot's responsibility; +# coupling this signal to Coroot would turn a component outage into a false +# cluster-down alarm. +# +# The URL comes from the dedicated Flux substitution input through a Secret +# reference, and curl consumes it from stdin configuration, so the bearer token +# does not appear in the PodSpec or curl's runtime argv. Local and unconfigured +# clusters use the invalid fallback, and `|| true` keeps the Job from flapping +# when no external monitor is configured. The signal is the absence of pings at +# the external monitor, not the Job exit code. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cluster-heartbeat + namespace: observability +spec: + schedule: "*/5 * * * *" + concurrencyPolicy: Forbid + startingDeadlineSeconds: 30 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 120 + template: + metadata: + labels: + app: cluster-heartbeat + platform-heartbeat: "true" + spec: + restartPolicy: Never + automountServiceAccountToken: false + priorityClassName: platform-critical + securityContext: + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: heartbeat + image: docker.io/curlimages/curl:8.21.0@sha256:7c12af72ceb38b7432ab85e1a265cff6ae58e06f95539d539b654f2cfa64bb13 + command: + - /bin/sh + - -c + args: + - >- + printf 'url = "%s"\n' "$HEARTBEAT_URL" | + curl --config - -sf --max-time 10 + --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused + || true + env: + - name: HEARTBEAT_URL + valueFrom: + secretKeyRef: + name: cluster-heartbeat + key: url + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + capabilities: + drop: + - ALL + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 50m + memory: 32Mi diff --git a/k8s/bases/apps/cluster-heartbeat/kustomization.yaml b/k8s/bases/apps/cluster-heartbeat/kustomization.yaml new file mode 100644 index 0000000..75a3a4f --- /dev/null +++ b/k8s/bases/apps/cluster-heartbeat/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# The apps Flux layer depends on infrastructure. Keeping the heartbeat here +# ensures the scoped observability deny-policy replacement is ready before the +# purpose-labelled workload starts during an existing-cluster upgrade. +resources: + - cilium-network-policy.yaml + - cron-job.yaml + - secret.yaml diff --git a/k8s/bases/apps/cluster-heartbeat/secret.yaml b/k8s/bases/apps/cluster-heartbeat/secret.yaml new file mode 100644 index 0000000..7b9cf3f --- /dev/null +++ b/k8s/bases/apps/cluster-heartbeat/secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: cluster-heartbeat + namespace: observability +type: Opaque +stringData: + # Flux substitutes the encrypted cluster variable here. The CronJob reads the + # resulting Secret so the bearer URL never appears in its PodSpec or argv. + url: "${cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}" diff --git a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml index d775742..d8e6b5c 100644 --- a/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml +++ b/k8s/bases/infrastructure/controllers/coroot/cilium-network-policy.yaml @@ -4,7 +4,12 @@ metadata: name: allow-coroot namespace: observability spec: - endpointSelector: {} + endpointSelector: + matchExpressions: + - key: platform-heartbeat + operator: NotIn + values: + - "true" ingress: # Coroot, its bundled stores, its agents, and the audit forwarder exchange # telemetry inside this namespace. diff --git a/k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml b/k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml new file mode 100644 index 0000000..d3302e0 --- /dev/null +++ b/k8s/components/observability-coroot/isolate-heartbeat/kustomization.yaml @@ -0,0 +1,100 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +# Only Coroot profiles stage the purpose-labelled heartbeat. Exclude that label +# from the generated namespace policies here, then replace both boundaries with +# narrower policies so default profiles keep their global zero-trust selectors. +patches: + - target: + group: kyverno.io + version: v1 + kind: ClusterPolicy + name: add-default-deny + patch: | + - op: test + path: /spec/rules/0/name + value: generate-default-deny + - op: test + path: /spec/rules/0/exclude/any/0/resources/names/2 + value: kube-node-lease + - op: add + path: /spec/rules/0/exclude/any/0/resources/names/- + value: observability + - op: test + path: /spec/rules/1/name + value: generate-allow-dns + - op: test + path: /spec/rules/1/exclude/any/0/resources/names/2 + value: kube-node-lease + - op: add + path: /spec/rules/1/exclude/any/0/resources/names/- + value: observability + - op: add + path: /spec/rules/- + value: + name: generate-default-deny-observability + match: + any: + - resources: + kinds: + - Namespace + names: + - observability + generate: + generateExisting: true + apiVersion: cilium.io/v2 + kind: CiliumNetworkPolicy + name: default-deny + synchronize: true + namespace: "{{request.object.metadata.name}}" + data: + spec: + endpointSelector: + matchExpressions: + - key: platform-heartbeat + operator: NotIn + values: + - "true" + enableDefaultDeny: + ingress: true + egress: true + ingressDeny: + - {} + egressDeny: + - {} + - op: add + path: /spec/rules/- + value: + name: generate-allow-dns-observability + match: + any: + - resources: + kinds: + - Namespace + names: + - observability + generate: + generateExisting: true + apiVersion: cilium.io/v2 + kind: CiliumNetworkPolicy + name: allow-dns + synchronize: true + namespace: "{{request.object.metadata.name}}" + data: + spec: + endpointSelector: + matchExpressions: + - key: platform-heartbeat + operator: NotIn + values: + - "true" + egress: + - toEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP diff --git a/k8s/providers/docker/apps-coroot/kustomization.yaml b/k8s/providers/docker/apps-coroot/kustomization.yaml index 03fb25b..f333d52 100644 --- a/k8s/providers/docker/apps-coroot/kustomization.yaml +++ b/k8s/providers/docker/apps-coroot/kustomization.yaml @@ -3,5 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../apps/ + - ../../../bases/apps/cluster-heartbeat/ components: - ../../../components/observability-coroot/remove-opencost-references/ diff --git a/k8s/providers/docker/infrastructure-coroot/kustomization.yaml b/k8s/providers/docker/infrastructure-coroot/kustomization.yaml index 42b0858..4753d03 100644 --- a/k8s/providers/docker/infrastructure-coroot/kustomization.yaml +++ b/k8s/providers/docker/infrastructure-coroot/kustomization.yaml @@ -6,4 +6,5 @@ resources: - ../../../bases/infrastructure/coroot/ components: - ../../../components/observability-coroot/allow-host-access/ + - ../../../components/observability-coroot/isolate-heartbeat/ - ../../../components/observability-coroot/remove-opencost-references/ diff --git a/k8s/providers/hetzner/apps-coroot/kustomization.yaml b/k8s/providers/hetzner/apps-coroot/kustomization.yaml index 03fb25b..f333d52 100644 --- a/k8s/providers/hetzner/apps-coroot/kustomization.yaml +++ b/k8s/providers/hetzner/apps-coroot/kustomization.yaml @@ -3,5 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ../apps/ + - ../../../bases/apps/cluster-heartbeat/ components: - ../../../components/observability-coroot/remove-opencost-references/ diff --git a/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml b/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml index 3b4c311..08c7cbd 100644 --- a/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure-coroot/kustomization.yaml @@ -7,6 +7,7 @@ resources: - ../../../bases/infrastructure/audit-log-forwarder/ components: - ../../../components/observability-coroot/allow-host-access/ + - ../../../components/observability-coroot/isolate-heartbeat/ - ../../../components/observability-coroot/remove-opencost-references/ patches: - path: patches/add-webhook-alerting.yaml diff --git a/scripts/validate-observability-option.sh b/scripts/validate-observability-option.sh index 5350ba5..fbf6957 100755 --- a/scripts/validate-observability-option.sh +++ b/scripts/validate-observability-option.sh @@ -6,6 +6,7 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" tmp_dir="$(mktemp -d)" trap 'rm -rf "${tmp_dir}"' EXIT +# Renders a repository-relative profile independently of the caller's working directory. render() { local source_path="$1" local output_path="$2" @@ -16,6 +17,7 @@ render() { fi } +# Counts one exact kind/namespace/name identity in a multi-document render. resource_count() { local rendered_path="$1" local kind="$2" @@ -30,6 +32,7 @@ resource_count() { '[.[] | select(type == "object" and .kind == $kind and (.metadata.namespace // "") == $namespace and .metadata.name == $name)] | length' } +# Counts rendered resources owned by the OpenCost namespace. opencost_resource_count() { local rendered_path="$1" @@ -37,6 +40,7 @@ opencost_resource_count() { jq -s '[.[] | select(type == "object" and ((.kind == "Namespace" and .metadata.name == "opencost") or .metadata.namespace == "opencost"))] | length' } +# Counts rendered resources whose serialized content still references OpenCost. opencost_reference_count() { local rendered_path="$1" @@ -44,6 +48,7 @@ opencost_reference_count() { jq -s '[.[] | select(type == "object" and (tojson | test("opencost"; "i")))] | length' } +# Counts Grafana data sources that target the retired Loki service. loki_datasource_count() { local rendered_path="$1" @@ -54,6 +59,7 @@ loki_datasource_count() { select(.type == "loki" or ((.url // "") | test("loki\\.monitoring\\.svc")))] | length' } +# Counts rendered resources whose serialized content still references Loki. loki_service_reference_count() { local rendered_path="$1" @@ -61,6 +67,7 @@ loki_service_reference_count() { jq -s '[.[] | select(type == "object" and (tojson | test("loki\\.monitoring\\.svc")))] | length' } +# Fails closed when an exact rendered identity is missing or duplicated. assert_resource_count() { local rendered_path="$1" local kind="$2" @@ -76,6 +83,7 @@ assert_resource_count() { fi } +# Returns the source path for one uniquely rendered Flux Kustomization. flux_path() { local rendered_path="$1" local name="$2" @@ -87,6 +95,7 @@ flux_path() { if length == 1 then .[0].spec.path else "invalid-count:\(length)" end' } +# Fails when a Flux Kustomization does not point at the expected profile path. assert_flux_path() { local rendered_path="$1" local name="$2" @@ -100,16 +109,438 @@ assert_flux_path() { fi } +# Verifies the reconciliation edge and wait boundary that keep a later Flux +# payload from starting before its prerequisite reports ready. +assert_flux_dependency() { + local rendered_path="$1" + local name="$2" + local dependency="$3" + local matches + + matches="$( + yq eval-all -o=json '.' "${rendered_path}" | + jq -s --arg name "${name}" --arg dependency "${dependency}" '[.[] | + select(.kind == "Kustomization" and .metadata.namespace == "flux-system" and .metadata.name == $name) | + select(.spec.dependsOn == [{"name":$dependency}]) | + select(.spec.wait == true)] | length' + )" + if [[ "${matches}" != "1" ]]; then + echo "Flux Kustomization/${name} must wait for ${dependency} in ${rendered_path}" >&2 + return 1 + fi +} + +# Verifies that the default profiles render no Coroot-owned resources. assert_default_off() { local rendered_path="$1" assert_resource_count "${rendered_path}" HelmRelease observability coroot-operator 0 assert_resource_count "${rendered_path}" Coroot observability coroot 0 assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0 + assert_resource_count "${rendered_path}" CronJob observability cluster-heartbeat 0 + assert_resource_count "${rendered_path}" Secret observability cluster-heartbeat 0 assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 0 + assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 0 assert_resource_count "${rendered_path}" HTTPRoute observability coroot 0 } +# Verifies that the heartbeat workload reconciles only in the apps layer, +# after infrastructure has replaced the observability namespace's generated +# deny policies with the scoped selectors. +assert_heartbeat_staged_after_policy_exclusion() { + local controllers_rendered_path="$1" + local apps_rendered_path="$2" + + assert_resource_count "${controllers_rendered_path}" CronJob observability cluster-heartbeat 0 + assert_resource_count "${controllers_rendered_path}" Secret observability cluster-heartbeat 0 + assert_resource_count "${controllers_rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 0 + assert_resource_count "${apps_rendered_path}" CronJob observability cluster-heartbeat 1 + assert_resource_count "${apps_rendered_path}" Secret observability cluster-heartbeat 1 + assert_resource_count "${apps_rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 1 +} + +# Executes the workflow's real heartbeat preflight with bounded fixture URLs. +run_bootstrap_heartbeat_preflight() { + local bootstrap_workflow="$1" + local cluster_heartbeat_url="$2" + local alertmanager_heartbeat_url="$3" + local validation_script + + validation_script="$( + sed -n '/# The dedicated Coroot heartbeat has an exact hc-ping.com egress/,/^[[:space:]]*fill_base()/p' "${bootstrap_workflow}" | + sed '$d' + )" + env \ + CLUSTER_HEARTBEAT_URL="${cluster_heartbeat_url}" \ + ALERTMANAGER_HEARTBEAT_URL="${alertmanager_heartbeat_url}" \ + bash -c "set -euo pipefail +${validation_script}" >/dev/null 2>&1 +} + +# Proves that bootstrap accepts two canonical checks while rejecting aliases, +# alternate healthchecks.io operations, and noncanonical check identifiers. +assert_bootstrap_heartbeat_identity_contract() { + local bootstrap_workflow="$1" + local cluster_url='https://hc-ping.com/11111111-1111-4111-8111-111111111111' + local alertmanager_url='https://hc-ping.com/22222222-2222-4222-8222-222222222222' + local cluster_alias + local alertmanager_alias + local -a cluster_aliases=( + "${cluster_url}#cluster" + "${cluster_url}?source=cluster" + "${cluster_url}/" + "${cluster_url}/start" + "${cluster_url}/fail" + "${cluster_url}/log" + "${cluster_url}/1" + 'https://HC-PING.com/11111111-1111-4111-8111-111111111111' + 'https://hc-ping.com:443/11111111-1111-4111-8111-111111111111' + 'https://hc-ping.com/project-slug/check-slug' + 'https://hc-ping.com/AAAAAAAA-AAAA-4AAA-8AAA-AAAAAAAAAAAA' + ) + local -a alertmanager_aliases=( + "${cluster_url}" + "${cluster_url}#alertmanager" + "${cluster_url}?source=alertmanager" + "${cluster_url}/" + "${cluster_url}/start" + "${cluster_url}/fail" + "${cluster_url}/log" + "${cluster_url}/1" + 'https://HC-PING.com/11111111-1111-4111-8111-111111111111' + 'https://hc-ping.com:443/11111111-1111-4111-8111-111111111111' + 'https://user@hc-ping.com/11111111-1111-4111-8111-111111111111' + 'https://hc-ping.com/project-slug/check-slug' + ) + + if ! run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_url}" "${alertmanager_url}"; then + echo "bootstrap must accept two distinct canonical healthchecks.io checks" >&2 + return 1 + fi + if ! run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "" "${alertmanager_url}"; then + echo "bootstrap must keep an unset direct-cluster check valid" >&2 + return 1 + fi + for cluster_alias in "${cluster_aliases[@]}"; do + if run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_alias}" "${alertmanager_url}"; then + echo "bootstrap must reject every noncanonical direct-cluster check URL" >&2 + return 1 + fi + done + for alertmanager_alias in "${alertmanager_aliases[@]}"; do + if run_bootstrap_heartbeat_preflight "${bootstrap_workflow}" "${cluster_url}" "${alertmanager_alias}"; then + echo "bootstrap must reject every alias of the direct-cluster check" >&2 + return 1 + fi + done +} + +assert_bootstrap_heartbeat_identity_contract "${repo_root}/.github/workflows/bootstrap.yaml" + +# Validates the singleton workload, isolation selector, and exact egress contract +# rendered by each explicit Coroot profile. +assert_coroot_heartbeat_contract() { + local rendered_path="$1" + local controllers_rendered_path="$2" + local heartbeat_count + local cronjob_contract + local secret_contract + local heartbeat_egress_contract + local coroot_selector_contract + local heartbeat_selecting_policy_count + + heartbeat_count="$(resource_count "${rendered_path}" CronJob observability cluster-heartbeat)" + cronjob_contract="$( + yq eval-all -o=json '.' "${rendered_path}" | + jq -s '[.[] | + select(.kind == "CronJob" and .metadata.namespace == "observability" and .metadata.name == "cluster-heartbeat") | + select(.spec.schedule == "*/5 * * * *") | + select(.spec.concurrencyPolicy == "Forbid") | + select(.spec.startingDeadlineSeconds == 30) | + select(.spec.successfulJobsHistoryLimit == 1 and .spec.failedJobsHistoryLimit == 1) | + select(.spec.jobTemplate.spec.backoffLimit == 0 and .spec.jobTemplate.spec.activeDeadlineSeconds == 120) | + .spec.jobTemplate.spec.template as $template | + select($template.metadata.labels.app == "cluster-heartbeat") | + select($template.metadata.labels["platform-heartbeat"] == "true") | + $template.spec as $pod | + select($pod.restartPolicy == "Never" and $pod.automountServiceAccountToken == false) | + select($pod.priorityClassName == "platform-critical") | + select($pod.securityContext.runAsNonRoot == true and $pod.securityContext.runAsUser == 65532) | + select($pod.securityContext.seccompProfile.type == "RuntimeDefault") | + select($pod.containers | length == 1) | + $pod.containers[0] as $container | + select($container.name == "heartbeat") | + select($container.image == "docker.io/curlimages/curl:8.21.0@sha256:7c12af72ceb38b7432ab85e1a265cff6ae58e06f95539d539b654f2cfa64bb13") | + select($container.command == [ + "/bin/sh", + "-c" + ]) | + select($container.args == [ + "printf '\''url = \"%s\"\\n'\'' \"$HEARTBEAT_URL\" | curl --config - -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused || true" + ]) | + select($container.env == [{ + "name":"HEARTBEAT_URL", + "valueFrom":{"secretKeyRef":{"name":"cluster-heartbeat","key":"url"}} + }]) | + select($container.securityContext.allowPrivilegeEscalation == false) | + select($container.securityContext.readOnlyRootFilesystem == true) | + select($container.securityContext.runAsNonRoot == true and $container.securityContext.runAsUser == 65532 and $container.securityContext.runAsGroup == 65532) | + select($container.securityContext.capabilities.drop == ["ALL"]) | + select($container.resources == { + "requests":{"cpu":"5m","memory":"16Mi"}, + "limits":{"cpu":"50m","memory":"32Mi"} + })] | length' + )" + secret_contract="$( + yq eval-all -o=json '.' "${rendered_path}" | + jq -s '[.[] | + select(.kind == "Secret" and .metadata.namespace == "observability" and .metadata.name == "cluster-heartbeat") | + select(.type == "Opaque") | + select(.stringData == {"url":"${cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}"})] | length' + )" + heartbeat_egress_contract="$( + yq eval-all -o=json '.' "${rendered_path}" | + jq -s '[.[] | + select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-cluster-heartbeat") | + select(.spec == { + "endpointSelector":{"matchLabels":{"platform-heartbeat":"true"}}, + "enableDefaultDeny":{"ingress":true,"egress":true}, + "ingressDeny":[{}], + "egress":[ + { + "toFQDNs":[{"matchName":"hc-ping.com"}], + "toPorts":[{"ports":[{"port":"443","protocol":"TCP"}]}] + }, + { + "toEndpoints":[{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}], + "toPorts":[{ + "ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}], + "rules":{"dns":[{"matchName":"hc-ping.com"}]} + }] + } + ] + })] | length' + )" + coroot_selector_contract="$( + yq eval-all -o=json '.' "${controllers_rendered_path}" | + jq -s '[.[] | + select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-coroot") | + select(.spec.endpointSelector.matchExpressions == [{"key":"platform-heartbeat","operator":"NotIn","values":["true"]}]) | + select(all(.spec.egress[]?.toFQDNs[]?; .matchName != "hc-ping.com"))] | length' + )" + heartbeat_selecting_policy_count="$( + yq eval-all -o=json '.' "${rendered_path}" "${controllers_rendered_path}" | + jq -s ' + def selector_matches($labels): + (.spec.endpointSelector // {}) as $selector | + all(($selector.matchLabels // {} | to_entries[]); $labels[.key] == .value) and + all(($selector.matchExpressions // [])[]; + .key as $key | + ($labels[$key] // null) as $value | + if .operator == "In" then + $value != null and ((.values // []) | index($value) != null) + elif .operator == "NotIn" then + $value == null or ((.values // []) | index($value) == null) + elif .operator == "Exists" then + $value != null + elif .operator == "DoesNotExist" then + $value == null + else + false + end); + {"app":"cluster-heartbeat","platform-heartbeat":"true"} as $heartbeat_labels | + [.[] | + select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability") | + select(selector_matches($heartbeat_labels))] | length' + )" + + if [[ "${heartbeat_count}" != "1" || "${cronjob_contract}" != "1" || "${secret_contract}" != "1" ]]; then + echo "Coroot profile must render one hardened five-minute heartbeat with one Secret reference in ${rendered_path}" >&2 + return 1 + fi + if [[ "${heartbeat_egress_contract}" != "1" || "${coroot_selector_contract}" != "1" || "${heartbeat_selecting_policy_count}" != "1" ]]; then + echo "Coroot heartbeat must match only its one exact configured-host egress policy in ${rendered_path}" >&2 + return 1 + fi +} + +# Validates the effective heartbeat after Flux post-build substitution. +assert_coroot_heartbeat_substitution() { + local rendered_path="$1" + local substituted_path + local heartbeat_url="https://hc-ping.com/example-heartbeat" + local heartbeat_host + local heartbeat_token='$' + local rendered_line + local rendered_prefix + local rendered_suffix + local cronjob_matches + local secret_matches + local policy_matches + + substituted_path="${tmp_dir}/$(basename "${rendered_path}" .yaml)-heartbeat-substituted.yaml" + heartbeat_token+='{cluster_heartbeat_url:=https://example.invalid/no-cluster-heartbeat-configured}' + heartbeat_host="${heartbeat_url#https://}" + heartbeat_host="${heartbeat_host%%/*}" + while IFS= read -r rendered_line || [[ -n "${rendered_line}" ]]; do + if [[ "${rendered_line}" == *"${heartbeat_token}"* ]]; then + rendered_prefix="${rendered_line%%"${heartbeat_token}"*}" + rendered_suffix="${rendered_line#*"${heartbeat_token}"}" + rendered_line="${rendered_prefix}${heartbeat_url}${rendered_suffix}" + fi + printf '%s\n' "${rendered_line}" + done <"${rendered_path}" >"${substituted_path}" + + cronjob_matches="$( + yq eval-all -o=json '.' "${substituted_path}" | + jq -s --arg heartbeat_url "${heartbeat_url}" '[.[] | + select(.kind == "CronJob" and .metadata.namespace == "observability" and .metadata.name == "cluster-heartbeat") | + .spec.jobTemplate.spec.template.spec.containers[0] as $container | + select($container.command == ["/bin/sh", "-c"]) | + select($container.args == [ + "printf '\''url = \"%s\"\\n'\'' \"$HEARTBEAT_URL\" | curl --config - -sf --max-time 10 --retry 3 --retry-delay 2 --retry-all-errors --retry-connrefused || true" + ]) | + select($container.env == [{ + "name":"HEARTBEAT_URL", + "valueFrom":{"secretKeyRef":{"name":"cluster-heartbeat","key":"url"}} + }])] | length' + )" + secret_matches="$( + yq eval-all -o=json '.' "${substituted_path}" | + jq -s --arg heartbeat_url "${heartbeat_url}" '[.[] | + select(.kind == "Secret" and .metadata.namespace == "observability" and .metadata.name == "cluster-heartbeat") | + select(.stringData.url == $heartbeat_url)] | length' + )" + policy_matches="$( + yq eval-all -o=json '.' "${substituted_path}" | + jq -s --arg heartbeat_host "${heartbeat_host}" '[.[] | + select(.kind == "CiliumNetworkPolicy" and .metadata.namespace == "observability" and .metadata.name == "allow-cluster-heartbeat") | + select(any(.spec.egress[]?.toFQDNs[]?; .matchName == $heartbeat_host)) | + select(any(.spec.egress[]?.toPorts[]?.rules.dns[]?; .matchName == $heartbeat_host))] | length' + )" + + if [[ "${cronjob_matches}" != "1" || "${secret_matches}" != "1" || "${policy_matches}" != "1" ]]; then + echo "Flux-substituted heartbeat URL and exact policy host must agree in ${rendered_path}" >&2 + return 1 + fi +} + +# Validates that the default profile still produces the Watchdog rule. +assert_watchdog_enabled() { + local rendered_path="$1" + local matches + + matches="$( + yq eval-all -o=json '.' "${rendered_path}" | + jq -s '[.[] | + select(.kind == "HelmRelease" and .metadata.namespace == "monitoring" and .metadata.name == "kube-prometheus-stack") | + select(.spec.values.defaultRules.create == true) | + select((.spec.values.defaultRules.rules.general // true) == true) | + select((.spec.values.defaultRules.disabled.Watchdog // false) == false) | + select([.spec.values.alertmanager.config.receivers[]? | + select(.name == "heartbeat") | + select(.webhook_configs[0].url == "${alertmanager_heartbeat_url:=https://example.invalid/no-heartbeat-configured}")] | length == 1)] | length' + )" + if [[ "${matches}" != "1" ]]; then + echo "kube-prometheus-stack must keep Watchdog on the distinct Alertmanager pipeline monitor in ${rendered_path}" >&2 + return 1 + fi +} + +# Validates that only the observability namespace's generated policies defer +# the dedicated heartbeat label to its narrower workload policy. +assert_heartbeat_policy_exclusion() { + local rendered_path="$1" + local rule_name + local scoped_rule_name + local expected_spec + local matches + + for rule_name in generate-default-deny generate-allow-dns; do + matches="$( + yq eval-all -o=json '.' "${rendered_path}" | + jq -s --arg rule_name "${rule_name}" '[.[] | + select(.kind == "ClusterPolicy" and .metadata.name == "add-default-deny") | + .spec.rules[] | + select(.name == $rule_name) | + select(.exclude.any[0].resources.names | index("observability")) | + select(.generate.data.spec.endpointSelector == {})] | length' + )" + if [[ "${matches}" != "1" ]]; then + echo "add-default-deny/${rule_name} must retain the global guardrail and exclude observability for its scoped replacement in ${rendered_path}" >&2 + return 1 + fi + + scoped_rule_name="${rule_name}-observability" + if [[ "${rule_name}" == "generate-default-deny" ]]; then + expected_spec='{ + "endpointSelector":{"matchExpressions":[{"key":"platform-heartbeat","operator":"NotIn","values":["true"]}]}, + "enableDefaultDeny":{"ingress":true,"egress":true}, + "ingressDeny":[{}], + "egressDeny":[{}] + }' + else + expected_spec='{ + "endpointSelector":{"matchExpressions":[{"key":"platform-heartbeat","operator":"NotIn","values":["true"]}]}, + "egress":[{ + "toEndpoints":[{"matchLabels":{"k8s:io.kubernetes.pod.namespace":"kube-system","k8s-app":"kube-dns"}}], + "toPorts":[{"ports":[{"port":"53","protocol":"UDP"},{"port":"53","protocol":"TCP"}]}] + }] + }' + fi + matches="$( + yq eval-all -o=json '.' "${rendered_path}" | + jq -s --arg rule_name "${scoped_rule_name}" --argjson expected_spec "${expected_spec}" '[.[] | + select(.kind == "ClusterPolicy" and .metadata.name == "add-default-deny") | + .spec.rules[] | + select(.name == $rule_name) | + select(.match.any[0].resources.kinds == ["Namespace"]) | + select(.match.any[0].resources.names == ["observability"]) | + select(.generate.data.spec == $expected_spec)] | length' + )" + if [[ "${matches}" != "1" ]]; then + echo "add-default-deny/${scoped_rule_name} must retain its complete scoped policy body in ${rendered_path}" >&2 + return 1 + fi + done +} + +# Verifies that default profiles retain the global deny and DNS selectors. +assert_heartbeat_policy_exclusion_absent() { + local rendered_path="$1" + local rule_name + local matches + + for rule_name in generate-default-deny generate-allow-dns; do + matches="$( + yq eval-all -o=json '.' "${rendered_path}" | + jq -s --arg rule_name "${rule_name}" '[.[] | + select(.kind == "ClusterPolicy" and .metadata.name == "add-default-deny") | + .spec.rules[] | + select(.name == $rule_name) | + select(((.exclude.any[0].resources.names // []) | index("observability")) == null) | + select(.generate.data.spec.endpointSelector == {})] | length' + )" + if [[ "${matches}" != "1" ]]; then + echo "default add-default-deny/${rule_name} must keep its global selector in ${rendered_path}" >&2 + return 1 + fi + done + + matches="$( + yq eval-all -o=json '.' "${rendered_path}" | + jq -s '[.[] | + select(.kind == "ClusterPolicy" and .metadata.name == "add-default-deny") | + .spec.rules[] | + select(.name == "generate-default-deny-observability" or .name == "generate-allow-dns-observability")] | length' + )" + if [[ "${matches}" != "0" ]]; then + echo "default add-default-deny must not contain Coroot heartbeat exceptions in ${rendered_path}" >&2 + return 1 + fi +} + +# Verifies that a default controller profile retains the complete OpenCost surface. assert_opencost_present() { local rendered_path="$1" @@ -120,6 +551,7 @@ assert_opencost_present() { assert_resource_count "${rendered_path}" CiliumNetworkPolicy opencost allow-opencost 1 } +# Verifies that a rendered profile contains no OpenCost-owned resources. assert_opencost_resources_absent() { local rendered_path="$1" local actual @@ -131,6 +563,7 @@ assert_opencost_resources_absent() { fi } +# Verifies the exact number of rendered resources that reference OpenCost. assert_opencost_reference_count() { local rendered_path="$1" local expected="$2" @@ -143,6 +576,7 @@ assert_opencost_reference_count() { fi } +# Verifies that both OpenCost resources and serialized references are absent. assert_opencost_absent() { local rendered_path="$1" local actual @@ -155,6 +589,7 @@ assert_opencost_absent() { fi } +# Verifies that Coroot reuses the hardened auth proxy without weakening it. assert_auth_proxy_with_coroot() { local default_rendered_path="$1" local opt_in_rendered_path="$2" @@ -188,6 +623,7 @@ assert_auth_proxy_with_coroot() { fi } +# Verifies the Coroot controller profile's SSO and access-control contract. assert_coroot_sso_controller_contract() { local rendered_path="$1" local route_contract @@ -260,6 +696,7 @@ assert_coroot_sso_controller_contract() { fi } +# Verifies whether a rendered Coroot project grants the expected admin role. assert_coroot_admin_role() { local rendered_path="$1" local expected="$2" @@ -284,6 +721,7 @@ assert_coroot_admin_role() { fi } +# Counts policy rules that omit the required namespace exclusion. policy_rules_without_namespace_exclusion() { local rendered_path="$1" local policy_name="$2" @@ -302,6 +740,7 @@ policy_rules_without_namespace_exclusion() { end' } +# Counts policy rules that carry a namespace exclusion. policy_rules_with_namespace_exclusion() { local rendered_path="$1" local policy_name="$2" @@ -320,6 +759,7 @@ policy_rules_with_namespace_exclusion() { end' } +# Verifies that every rule in a policy excludes the selected namespace. assert_namespace_excluded_from_all_policy_rules() { local rendered_path="$1" local policy_name="$2" @@ -333,6 +773,7 @@ assert_namespace_excluded_from_all_policy_rules() { fi } +# Verifies that no rule in a policy excludes the selected namespace. assert_namespace_not_excluded_from_policy() { local rendered_path="$1" local policy_name="$2" @@ -346,6 +787,7 @@ assert_namespace_not_excluded_from_policy() { fi } +# Verifies the exact namespace match count in one security exception. assert_security_exception_namespace_count() { local rendered_path="$1" local exception_name="$2" @@ -417,6 +859,10 @@ assert_flux_path "${prod_cluster_coroot}" bootstrap clusters/prod/bootstrap assert_flux_path "${prod_cluster_coroot}" infrastructure-controllers providers/hetzner/infrastructure-controllers-coroot assert_flux_path "${prod_cluster_coroot}" infrastructure providers/hetzner/infrastructure-coroot assert_flux_path "${prod_cluster_coroot}" apps providers/hetzner/apps-coroot +for rendered_path in "${local_cluster_coroot}" "${prod_cluster_coroot}"; do + assert_flux_dependency "${rendered_path}" infrastructure infrastructure-controllers + assert_flux_dependency "${rendered_path}" apps infrastructure +done # Cluster renders contain Flux pointers, so inspect the provider payloads those # pointers reconcile. This catches accidental activation in either layer. @@ -429,8 +875,10 @@ render k8s/providers/hetzner/apps/ "${prod_apps_default}" for rendered_path in \ "${local_controllers_default}" \ "${local_infrastructure_default}" \ + "${local_apps_default}" \ "${prod_controllers_default}" \ - "${prod_infrastructure_default}"; do + "${prod_infrastructure_default}" \ + "${prod_apps_default}"; do assert_default_off "${rendered_path}" done @@ -438,6 +886,7 @@ done # Only the explicit Coroot profiles retire it. for rendered_path in "${local_controllers_default}" "${prod_controllers_default}"; do assert_opencost_present "${rendered_path}" + assert_watchdog_enabled "${rendered_path}" done for rendered_path in "${local_infrastructure_default}" "${prod_infrastructure_default}"; do assert_opencost_resources_absent "${rendered_path}" @@ -447,6 +896,7 @@ for rendered_path in "${local_infrastructure_default}" "${prod_infrastructure_de assert_security_exception_namespace_count "${rendered_path}" infrastructure-privileged observability 0 assert_security_exception_namespace_count "${rendered_path}" controller-rbac observability 0 assert_security_exception_namespace_count "${rendered_path}" service-account-tokens observability 0 + assert_heartbeat_policy_exclusion_absent "${rendered_path}" done assert_opencost_reference_count "${local_apps_default}" 1 assert_opencost_reference_count "${prod_apps_default}" 1 @@ -487,6 +937,9 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do fi done +assert_heartbeat_staged_after_policy_exclusion "${docker_controllers}" "${docker_apps}" +assert_heartbeat_staged_after_policy_exclusion "${hetzner_controllers}" "${hetzner_apps}" + assert_auth_proxy_with_coroot "${local_controllers_default}" "${docker_controllers}" assert_auth_proxy_with_coroot "${prod_controllers_default}" "${hetzner_controllers}" @@ -494,11 +947,21 @@ for rendered_path in "${docker_controllers}" "${hetzner_controllers}"; do assert_resource_count "${rendered_path}" HelmRelease observability coroot-operator 1 assert_resource_count "${rendered_path}" Coroot observability coroot 0 assert_resource_count "${rendered_path}" HelmRelease observability audit-log-forwarder 0 + assert_resource_count "${rendered_path}" HelmRelease monitoring kube-prometheus-stack 1 assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-coroot 1 + assert_resource_count "${rendered_path}" CronJob observability cluster-heartbeat 0 + assert_resource_count "${rendered_path}" Secret observability cluster-heartbeat 0 + assert_resource_count "${rendered_path}" CiliumNetworkPolicy observability allow-cluster-heartbeat 0 assert_opencost_absent "${rendered_path}" assert_coroot_sso_controller_contract "${rendered_path}" + assert_watchdog_enabled "${rendered_path}" done +assert_coroot_heartbeat_contract "${docker_apps}" "${docker_controllers}" +assert_coroot_heartbeat_substitution "${docker_apps}" +assert_coroot_heartbeat_contract "${hetzner_apps}" "${hetzner_controllers}" +assert_coroot_heartbeat_substitution "${hetzner_apps}" + for rendered_path in "${docker_infrastructure}" "${hetzner_infrastructure}"; do assert_namespace_excluded_from_all_policy_rules "${rendered_path}" add-security-context observability assert_namespace_excluded_from_all_policy_rules "${rendered_path}" validate-host-restrictions observability @@ -507,6 +970,7 @@ for rendered_path in "${docker_infrastructure}" "${hetzner_infrastructure}"; do assert_security_exception_namespace_count "${rendered_path}" controller-rbac observability 1 assert_security_exception_namespace_count "${rendered_path}" service-account-tokens observability 1 assert_security_exception_namespace_count "${rendered_path}" health-probes observability 0 + assert_heartbeat_policy_exclusion "${rendered_path}" done assert_resource_count "${docker_infrastructure}" HelmRelease observability coroot-operator 0 @@ -659,6 +1123,18 @@ for documented_boundary in \ "This profile is transitional." \ "removes OpenCost and its Headlamp" \ "retires the legacy Loki and Alloy log path" \ + "stages one hardened \`cluster-heartbeat\` CronJob" \ + "uses a separate optional \`CLUSTER_HEARTBEAT_URL\` secret" \ + "Never point both inputs at the same monitor" \ + "permits only \`hc-ping.com:443\`" \ + "Bootstrap accepts only a canonical" \ + "compares check identities" \ + "dependent apps layer stages the heartbeat only after" \ + "Secret-backed environment variable" \ + "stdin configuration" \ + "platform-critical priority" \ + "keep Watchdog unchanged" \ + "preserves a dead-man signal" \ "keeps kube-prometheus-stack" \ "Cost allocation is therefore unavailable" \ "reuses the existing encrypted webhook" \ @@ -666,7 +1142,7 @@ for documented_boundary in \ "permits only \`hooks.slack.com:443\`" \ "Per-alert notifications remain visible only in the Coroot UI" \ "Local / Docker Coroot stays notification-free" \ - "Kube-prometheus-stack keeps owning its alert rules" \ + "Kube-prometheus-stack keeps owning its remaining alert" \ "https://observability." \ "Dex-backed oauth2-proxy" \ "no direct Gateway route to the Coroot service" \ @@ -677,9 +1153,62 @@ for documented_boundary in \ fi done +if grep -R -n 'alertmanager_heartbeat_host' \ + "${repo_root}/.github/workflows/bootstrap.yaml" \ + "${repo_root}/docs" \ + "${repo_root}/k8s"; then + echo "Coroot heartbeat must not add an instance-owned host variable" >&2 + exit 1 +fi + +alerting_guide="${repo_root}/docs/dr/alerting.md" +for documented_heartbeat_boundary in \ + "Two independent dead-man checks" \ + "never give them the same ping URL" \ + "Watchdog stays enabled" \ + "cluster_heartbeat_url" \ + "The three secret inputs" \ + "\`hc-ping.com:443\`" \ + "Bootstrap compares" \ + "canonical check identities" \ + "dependent apps layer" \ + "while Watchdog preserves" \ + "curl as stdin configuration" \ + "Flux reconciliation alerting still depends on kube-prometheus-stack"; do + if ! grep -Fq "${documented_heartbeat_boundary}" "${alerting_guide}"; then + echo "alerting guide does not retain heartbeat boundary: ${documented_heartbeat_boundary}" >&2 + exit 1 + fi +done + +bootstrap_workflow="${repo_root}/.github/workflows/bootstrap.yaml" +for bootstrap_boundary in \ + "CLUSTER_HEARTBEAT_URL: \${{ secrets.CLUSTER_HEARTBEAT_URL }}" \ + "heartbeat_url_pattern='^https://hc-ping\\.com/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$'" \ + "heartbeat_host_pattern='^https?://([^/?#]*@)?hc-ping\\.com([:/?#]|$)'" \ + "alertmanager_heartbeat_url_lower=\"\$(printf '%s' \"\${ALERTMANAGER_HEARTBEAT_URL}\" | tr '[:upper:]' '[:lower:]')\"" \ + 'CLUSTER_HEARTBEAT_URL must be a canonical https://hc-ping.com/ URL or be unset' \ + 'ALERTMANAGER_HEARTBEAT_URL must be canonical when it uses hc-ping.com' \ + "cluster_heartbeat_check_id=\"\${BASH_REMATCH[1]}\"" \ + "alertmanager_heartbeat_check_id=\"\${BASH_REMATCH[1]}\"" \ + "[ \"\${cluster_heartbeat_check_id}\" = \"\${alertmanager_heartbeat_check_id}\" ]; then" \ + 'CLUSTER_HEARTBEAT_URL must use a different external check than ALERTMANAGER_HEARTBEAT_URL' \ + '.stringData.cluster_heartbeat_url = strenv(CLUSTER_HEARTBEAT_URL)'; do + if ! grep -Fq "${bootstrap_boundary}" "${bootstrap_workflow}"; then + echo "bootstrap workflow does not retain cluster-heartbeat boundary: ${bootstrap_boundary}" >&2 + exit 1 + fi +done + +if grep -Fq 'alertmanager_heartbeat_url' "${repo_root}/k8s/bases/apps/cluster-heartbeat/cron-job.yaml"; then + echo "Coroot CronJob must not share the Alertmanager pipeline monitor" >&2 + exit 1 +fi + if grep -R -E -n '(devantler|homelab|hooks\.slack\.com|hcloud|alertmanager_webhook_url)' \ "${repo_root}/k8s/bases/infrastructure/controllers/coroot" \ "${repo_root}/k8s/bases/infrastructure/coroot" \ + "${repo_root}/k8s/bases/apps/cluster-heartbeat" \ "${repo_root}/k8s/bases/infrastructure/audit-log-forwarder"; then echo "Coroot profile manifests contain an instance-specific value" >&2 exit 1