From 78cfa2c6c5dfc690615c83aa6ab1d48edcfa9599 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 27 Jul 2026 02:59:51 -0700 Subject: [PATCH] =?UTF-8?q?feat(attest):=20CoCo=20dev-mode=20stack=20?= =?UTF-8?q?=E2=80=94=20pinned=20manifests,=20KBS=20topology=20decision=20(?= =?UTF-8?q?#9213)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vendors the Confidential Containers Helm chart (dev-only shim override) and Trustee KBS kustomize manifests in no-TEE dev mode, with every version pinned in versions.json and drift-checked in CI. Records the Trustee/KBS-mediated topology decision and the plan for recording real launch measurements once SNP hardware arrives, so #8536 becomes a config apply plus measurement recording on real metal. Verified against a real local kind cluster: the chart installs, the kata-qemu-coco-dev RuntimeClass is created, and the KBS manifests apply cleanly. Pod execution itself needs a real amd64 host (both the KBS image and the kata-deploy dev-shim daemonset are amd64/s390x-only); the attested-run harness (sample attester path) was verified standalone since it needs no Kubernetes/Kata/KBS at all. --- k8s/coco-dev/.gitignore | 7 + k8s/coco-dev/README.md | 122 ++++++++++++++++++ k8s/coco-dev/helm/values-coco-dev.yaml | 24 ++++ k8s/coco-dev/kbs/base/deployment.yaml | 40 ++++++ k8s/coco-dev/kbs/base/kbs-config.toml | 36 ++++++ k8s/coco-dev/kbs/base/kustomization.yaml | 29 +++++ k8s/coco-dev/kbs/base/namespace.yaml | 4 + k8s/coco-dev/kbs/base/service.yaml | 12 ++ .../kbs/overlays/dev/kustomization.yaml | 23 ++++ k8s/coco-dev/kbs/overlays/dev/patch.yaml | 15 +++ k8s/coco-dev/scripts/down.sh | 15 +++ k8s/coco-dev/scripts/generate-kbs-keys.sh | 27 ++++ k8s/coco-dev/scripts/up.sh | 43 ++++++ k8s/coco-dev/versions.json | 15 +++ package.json | 3 +- scripts/check-coco-dev-versions-core.ts | 59 +++++++++ scripts/check-coco-dev-versions.ts | 35 +++++ .../unit/check-coco-dev-versions-core.test.ts | 93 +++++++++++++ 18 files changed, 601 insertions(+), 1 deletion(-) create mode 100644 k8s/coco-dev/.gitignore create mode 100644 k8s/coco-dev/README.md create mode 100644 k8s/coco-dev/helm/values-coco-dev.yaml create mode 100644 k8s/coco-dev/kbs/base/deployment.yaml create mode 100644 k8s/coco-dev/kbs/base/kbs-config.toml create mode 100644 k8s/coco-dev/kbs/base/kustomization.yaml create mode 100644 k8s/coco-dev/kbs/base/namespace.yaml create mode 100644 k8s/coco-dev/kbs/base/service.yaml create mode 100644 k8s/coco-dev/kbs/overlays/dev/kustomization.yaml create mode 100644 k8s/coco-dev/kbs/overlays/dev/patch.yaml create mode 100755 k8s/coco-dev/scripts/down.sh create mode 100755 k8s/coco-dev/scripts/generate-kbs-keys.sh create mode 100755 k8s/coco-dev/scripts/up.sh create mode 100644 k8s/coco-dev/versions.json create mode 100644 scripts/check-coco-dev-versions-core.ts create mode 100644 scripts/check-coco-dev-versions.ts create mode 100644 test/unit/check-coco-dev-versions-core.test.ts diff --git a/k8s/coco-dev/.gitignore b/k8s/coco-dev/.gitignore new file mode 100644 index 0000000000..2dac8fa33f --- /dev/null +++ b/k8s/coco-dev/.gitignore @@ -0,0 +1,7 @@ +# Generated locally by scripts/generate-kbs-keys.sh, exactly matching upstream Trustee's own +# kbs/config/kubernetes/.gitignore -- an ed25519 admin keypair and a dev workload secret, never real +# production key material, but still never committed: kustomize's secretGenerator needs the files to +# exist on disk at apply time, not at checkout time. +kbs/base/kbs.pem +kbs/base/kbs.key +kbs/overlays/dev/key.bin diff --git a/k8s/coco-dev/README.md b/k8s/coco-dev/README.md new file mode 100644 index 0000000000..6f34a45a5c --- /dev/null +++ b/k8s/coco-dev/README.md @@ -0,0 +1,122 @@ +# CoCo dev stack — metal prep (#9213, epic #8534) + +Everything about the eventual attested-evaluation stack *except the TEE itself* can be built and rehearsed +today, on ordinary hardware, using the [Confidential Containers](https://github.com/confidential-containers) +project's own no-hardware "dev/sample" mode. This directory is that rehearsal: committed manifests, pinned +versions, and a recorded topology decision, so that [#8536](https://github.com/JSONbored/loopover/issues/8536) +(the same stack on real AMD SEV-SNP metal) is a config apply plus measurement recording — no net-new design. + +## What's here + +``` +k8s/coco-dev/ + versions.json -- pinned CoCo Helm chart + Trustee KBS versions (single source of truth; + scripts/check-coco-dev-versions.ts fails CI if kustomization.yaml drifts + from this file) + helm/values-coco-dev.yaml -- dev-only override: disables the SNP/TDX/GPU shims, keeps only the no- + hardware `qemu-coco-dev` shim + kbs/base/, kbs/overlays/dev/ -- Trustee KBS kustomize manifests, vendored from + confidential-containers/trustee's own kbs/config/kubernetes/, digest- + pinned instead of tag-pinned + scripts/up.sh, down.sh -- idempotent bring-up/teardown (kind cluster + Helm chart + KBS) + scripts/generate-kbs-keys.sh -- generates the local-only KBS admin keypair + dev workload secret, + mirroring upstream's own deploy-kbs.sh; never committed (see .gitignore) +``` + +## What was actually verified, and where + +Run against a real local `kind` cluster (Apple Silicon, arm64 — this repo's own dev machine) during this +issue's implementation: + +- `kind create cluster` — a real Kubernetes control plane comes up. +- `helm upgrade --install coco oci://ghcr.io/confidential-containers/charts/confidential-containers --version + 0.21.0 -f helm/values-coco-dev.yaml` — installs cleanly; `kubectl get runtimeclass` shows + `kata-qemu-coco-dev` and its siblings created for real. +- `kubectl apply -k kbs/overlays/dev` — the KBS namespace, ConfigMap, Secrets, Service, and Deployment all + apply and the pod schedules. +- `npx tsx scripts/attested-backtest-run.ts --attester sample ...` (the attested-run harness, #9211) — runs + to completion and returns `{ "status": "attested", "attesterKind": "sample", ... }`, exit code 0. This path + needs no Kubernetes/Kata/KBS at all (that's the whole design point of the sample attester, see + `scripts/attested-backtest-run.ts`'s own header comment) and was verified standalone. + +**Not achievable on this dev machine, for a real and structural reason, not a bug:** both the published +Trustee KBS image and kata-deploy's own dev-shim daemonset images are built `amd64`/`s390x` only — no `arm64` +manifest exists. The KBS pod sits in `ImagePullBackOff` on this cluster +(`no match for platform in manifest: not found`), and `kata-as-coco-runtime`'s daemonset shows `0` ready, for +the identical reason. Loading a pre-pulled, emulated (`docker pull --platform linux/amd64`) copy of the image +directly into the kind node via `kind load docker-image` does **not** work around this: the CRI image service +still rejects the pull by platform even for content already resident on the node, regardless of +`imagePullPolicy`. This is the same class of environmental boundary as SNP hardware itself or `/dev/kvm` for +real Kata sandboxing (documented for those, e.g. in `scripts/verify-attested-run/README.md`) — a fact about +this machine's architecture, not something to route around. **Anyone running `scripts/up.sh` on a real amd64 +Linux host** (which is what production/#8536 actually targets) hits none of this: the images pull and run +natively. + +We deliberately did **not** add a recurring CI job to get a green run of the blocked half on GitHub-hosted +(amd64) runners, and did **not** stand up a self-hosted runner for it. `versions.json` changes rarely and +deliberately; the per-PR `coco-dev-versions:check` drift check already catches the one failure mode that +actually recurs (a version bumped in one file but not the other). A heavyweight, kind-cluster-plus-image-pull +job re-run on every push to main for a stack this static isn't proportional to that risk, and a self-hosted +runner is ongoing infrastructure and public-repo attack surface to secure indefinitely for the same modest +payoff. If a maintainer wants to fully exercise this stack, `scripts/up.sh` runs the identical, real commands +on any amd64 Linux host or VM (see `scripts/generate-kbs-keys.sh` for the one manual prerequisite). + +## Topology decision: Trustee/KBS-mediated, not direct attestation-agent → verifier-CLI + +Two ways a guest workload's attestation evidence can reach a verifier: + +1. **Trustee/KBS-mediated** (what's vendored here): the guest's attestation-agent talks to a KBS, which runs + the verification (via its built-in Attestation Service) and, only on success, releases a secret the guest + needs to do its job. The **secret release is gated on verification** — a guest that can't attest gets + nothing, enforced by infrastructure the guest itself can't bypass. +2. **Direct attestation-agent → verifier-CLI**: the guest (or something watching it) hands its raw + attestation report to `scripts/verify-attested-run.ts` (#9212) directly; nothing is gated on the result + unless something else chooses to act on the CLI's exit code. + +**Decision: KBS-mediated.** Reasons: + +- **Real secrets exist on this path.** The attested workload needs an actual DB credential / signing key to + do its job (record an attested-run outcome, sign a result) — this is not attestation for attestation's own + sake. A KBS is *specifically* the component whose job is "hold a secret, release it only after verifying + the requester," which is exactly the shape of that need. A direct CLI-only flow would still need something + else to hold and gate that secret — reinventing a narrower, unaudited KBS instead of using the project's + own maintained one. +- **Fail-closed by construction, not by convention.** Under direct-verifier-CLI, "gate on the result" is a + policy someone has to remember to implement and can't easily audit from outside. Under KBS-mediation, an + unattested guest has no secret and therefore cannot proceed, full stop — the gate is structural, not just + a check someone remembered to call. +- **`scripts/verify-attested-run.ts` (#9212) is not made redundant by this decision.** It stays the tool a + *third party*, outside LoopOver's own infrastructure and holding none of its secrets, uses to independently + re-verify a published envelope after the fact — a different audience and a different trust question + ("should I, an outsider, believe this run happened") than "should the KBS release this specific secret to + this specific guest right now." Both paths matter; they answer different questions for different parties. +- **Cost is bounded and already paid.** The KBS is one more component to deploy and pin (done here), but + Trustee is the project's own maintained, documented answer to exactly this problem — not a bespoke one this + repo would otherwise have to build and keep secure itself. + +## Recording real launch measurements once SNP hardware arrives (#8536) + +Today, `scripts/attested-backtest-run.ts --attester sample` never claims a measurement (the sample attester's +whole point is being self-labeling and unable to lie about being real evidence — see +`scripts/verify-attested-run-core.ts`'s `sample_attestation` rejection). On real hardware: + +1. Deploy the **same** `k8s/coco-dev/kbs/` manifests plus a values override enabling the real `qemu-snp` shim + in place of `qemu-coco-dev` (mechanically: swap which shim is `enabled: true` in a values file structured + like `helm/values-coco-dev.yaml` — the chart, version, and every other setting stay identical). +2. Launch the pinned replay-runner image (`scripts/replay-runner/Dockerfile`, #9214) under the real + `kata-qemu-snp` RuntimeClass. The hypervisor computes the launch measurement from the actual guest firmware + + kernel + initrd + rootfs it launched — this is **not** a value LoopOver computes or asserts; it is what + makes measurement pinning meaningful evidence rather than a self-report. +3. Capture that measurement from the first genuine attestation report obtained via `scripts/snp-attester.ts` + (#9211) — read `attestationReport`'s `measurement` field (see `scripts/verify-attested-run-report.ts` for + the exact offset) — and record it as `expected.json`'s `measurement` field for + `scripts/verify-attested-run.ts` (#9212), and/or as a new pinned constant wherever the live gate's own + fail-closed check (`--runtime-claim tee`, `scripts/attested-backtest-run-core.ts`) needs to compare against + it. +4. Re-derive that measurement on every subsequent image rebuild by re-running step 2 against the newly built, + digest-pinned image (`scripts/replay-runner-image-manifest.json`, #9214) and diffing — an *unexpected* + measurement change with no corresponding image change is itself a signal worth investigating, not + something to silently re-pin. + +This plan needs no new design work once hardware exists — it is "run the thing that's already built, once, +under the real shim, and copy a number out of a report." diff --git a/k8s/coco-dev/helm/values-coco-dev.yaml b/k8s/coco-dev/helm/values-coco-dev.yaml new file mode 100644 index 0000000000..eafdb5e1f3 --- /dev/null +++ b/k8s/coco-dev/helm/values-coco-dev.yaml @@ -0,0 +1,24 @@ +# Helm values override for the confidential-containers chart (see ../versions.json for the pinned +# chart version) -- no-TEE dev mode only. The chart's own default values.yaml enables the real SNP/TDX/GPU +# shims unconditionally; this override explicitly disables every one of them and enables only +# `qemu-coco-dev`, the project's own no-hardware "sample and mocking implementations" shim (see +# https://github.com/confidential-containers/confidential-containers/blob/main/guides/coco-dev.md). Applying +# this same chart+version on real SNP metal (#8536) means dropping this override file, not changing anything +# else -- that's the whole point of pinning the topology here first. +kata-as-coco-runtime: + enabled: true + shims: + qemu-snp: + enabled: false + qemu-tdx: + enabled: false + qemu-coco-dev: + enabled: true + qemu-coco-dev-runtime-rs: + enabled: false + qemu-nvidia-gpu-snp: + enabled: false + qemu-nvidia-gpu-tdx: + enabled: false + defaultShim: + amd64: qemu-coco-dev diff --git a/k8s/coco-dev/kbs/base/deployment.yaml b/k8s/coco-dev/kbs/base/deployment.yaml new file mode 100644 index 0000000000..a0953edcad --- /dev/null +++ b/k8s/coco-dev/kbs/base/deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kbs +spec: + selector: + matchLabels: + app: kbs + replicas: 1 + template: + metadata: + labels: + app: kbs + spec: + containers: + - name: kbs + image: kbs-container-image + ports: + - containerPort: 8080 + imagePullPolicy: Always + command: + - /usr/local/bin/kbs + - --config-file + - /etc/kbs/kbs-config.toml + volumeMounts: + - name: kbs-auth-public-key + mountPath: /kbs/ + - name: kbs-config + mountPath: /etc/kbs/ + - name: policy-volume + mountPath: /opt/confidential-containers/kbs/ + volumes: + - name: kbs-auth-public-key + secret: + secretName: kbs-auth-public-key + - name: kbs-config + configMap: + name: kbs-config + - name: policy-volume + emptyDir: {} diff --git a/k8s/coco-dev/kbs/base/kbs-config.toml b/k8s/coco-dev/kbs/base/kbs-config.toml new file mode 100644 index 0000000000..c234912c10 --- /dev/null +++ b/k8s/coco-dev/kbs/base/kbs-config.toml @@ -0,0 +1,36 @@ +[http_server] +sockets = ["0.0.0.0:8080"] +# Dev mode only -- no TLS termination. A real deployment (#8536) needs a certificate solution +# (e.g. cert-manager) in front of this. +insecure_http = true + +[attestation_token] +insecure_header_jwk = true + +[attestation_service] +type = "coco_as_builtin" + +[attestation_service.attestation_token_broker] +duration_min = 5 + +[attestation_service.rvps_config] +type = "BuiltIn" + +[admin] +authorization_mode = "AuthenticatedAuthorization" + +[admin.authentication.bearer_jwt] +identity_providers = [{ public_key_uri = "/kbs/kbs.pem" }] + +[admin.authorization.regex_acl] +acls = [{ role = "admin", allowed_endpoints = "^/kbs/.+$" }] + +[[plugins]] +name = "resource" +storage_backend_type = "kvstorage" + +[storage_backend] +storage_type = "LocalFs" + +[storage_backend.backends.local_fs] +dir_path = "/opt/confidential-containers/kbs" diff --git a/k8s/coco-dev/kbs/base/kustomization.yaml b/k8s/coco-dev/kbs/base/kustomization.yaml new file mode 100644 index 0000000000..a50e4ef29f --- /dev/null +++ b/k8s/coco-dev/kbs/base/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: coco-tenant + +# Digest-pinned, not tag-pinned (upstream's own base/kustomization.yaml uses `newTag:`) -- a tag can be +# repointed by the upstream project, a digest cannot. Keep this in sync with ../../versions.json's +# `trusteeKbs.digest`; scripts/check-coco-dev-versions.ts fails CI if the two ever drift apart. +images: +- name: kbs-container-image + newName: ghcr.io/confidential-containers/key-broker-service + digest: sha256:16926d905621e94dadadb95fd0794dddae8ee30bdf71c71940c6c8fe25087119 + +resources: +- namespace.yaml +- deployment.yaml +- service.yaml + +# KBS configuration. +configMapGenerator: +- files: + - kbs-config.toml + name: kbs-config + +# KBS auth public key. Generated locally by scripts/generate-kbs-keys.sh, never committed -- see +# ../../.gitignore. Kustomize needs the file to exist at apply time, not at checkout time. +secretGenerator: +- files: + - kbs.pem + name: kbs-auth-public-key diff --git a/k8s/coco-dev/kbs/base/namespace.yaml b/k8s/coco-dev/kbs/base/namespace.yaml new file mode 100644 index 0000000000..7369170866 --- /dev/null +++ b/k8s/coco-dev/kbs/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: coco-tenant diff --git a/k8s/coco-dev/kbs/base/service.yaml b/k8s/coco-dev/kbs/base/service.yaml new file mode 100644 index 0000000000..297d170e27 --- /dev/null +++ b/k8s/coco-dev/kbs/base/service.yaml @@ -0,0 +1,12 @@ +# Service to expose the KBS. +apiVersion: v1 +kind: Service +metadata: + name: kbs +spec: + selector: + app: kbs + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 diff --git a/k8s/coco-dev/kbs/overlays/dev/kustomization.yaml b/k8s/coco-dev/kbs/overlays/dev/kustomization.yaml new file mode 100644 index 0000000000..36c59112b8 --- /dev/null +++ b/k8s/coco-dev/kbs/overlays/dev/kustomization.yaml @@ -0,0 +1,23 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: coco-tenant + +resources: +- ../../base + +patches: +- path: patch.yaml + target: + group: apps + kind: Deployment + name: kbs + version: v1 + +# Store keys that KBS will release to workloads after attestation: +# kbs:///reponame/workload_key/key.bin +# +# Generated locally by scripts/generate-kbs-keys.sh, never committed -- see ../../../.gitignore. +secretGenerator: +- files: + - key.bin + name: keys diff --git a/k8s/coco-dev/kbs/overlays/dev/patch.yaml b/k8s/coco-dev/kbs/overlays/dev/patch.yaml new file mode 100644 index 0000000000..c7f46e41fd --- /dev/null +++ b/k8s/coco-dev/kbs/overlays/dev/patch.yaml @@ -0,0 +1,15 @@ +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + name: keys + # Store keys that KBS will release to workloads after attestation: + # kbs:///reponame/workload_key/key.bin + # + # Note: The path should prefix kbs-config's repository_description.dir_path: /opt/confidential-containers/kbs/repository + mountPath: /opt/confidential-containers/kbs/repository/reponame/workload_key/ +- op: add + path: /spec/template/spec/volumes/- + value: + name: keys + secret: + secretName: keys diff --git a/k8s/coco-dev/scripts/down.sh b/k8s/coco-dev/scripts/down.sh new file mode 100755 index 0000000000..b2eadb9df9 --- /dev/null +++ b/k8s/coco-dev/scripts/down.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Idempotent teardown for ./up.sh -- deletes the whole `kind` cluster (the Helm release, the KBS deployment, +# and every namespace all go with it, since they exist only inside that disposable cluster). Never touches +# the locally generated key material under kbs/ -- re-running up.sh reuses the same keys, matching upstream +# Trustee's own deploy-kbs.sh behavior of only ever generating a key that's missing. +set -euo pipefail + +CLUSTER_NAME="${COCO_DEV_KIND_CLUSTER:-coco-dev}" + +if kind get clusters 2>/dev/null | grep -qx "${CLUSTER_NAME}"; then + echo "down: deleting kind cluster '${CLUSTER_NAME}'" + kind delete cluster --name "${CLUSTER_NAME}" +else + echo "down: kind cluster '${CLUSTER_NAME}' does not exist, nothing to do" +fi diff --git a/k8s/coco-dev/scripts/generate-kbs-keys.sh b/k8s/coco-dev/scripts/generate-kbs-keys.sh new file mode 100755 index 0000000000..30eb91a1dc --- /dev/null +++ b/k8s/coco-dev/scripts/generate-kbs-keys.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Generate the local-only KBS admin keypair and dev workload secret, exactly matching upstream Trustee's +# own kbs/config/kubernetes/deploy-kbs.sh procedure. Idempotent: skips generation for any file that already +# exists, so re-running never rotates a key silently. None of this is real production key material -- it is +# a fresh ed25519 admin keypair authorizing calls against a throwaway `kind` cluster, and a dev workload +# secret KBS will release to an attested guest. See ../.gitignore: none of these three files are committed. +set -euo pipefail + +COCO_DEV_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +KBS_KEY="${COCO_DEV_DIR}/kbs/base/kbs.key" +KBS_CERT="${COCO_DEV_DIR}/kbs/base/kbs.pem" +WORKLOAD_KEY="${COCO_DEV_DIR}/kbs/overlays/dev/key.bin" + +if [[ -f "${KBS_CERT}" ]]; then + echo "generate-kbs-keys: ${KBS_CERT} already exists, leaving it in place" +else + openssl genpkey -algorithm ed25519 -out "${KBS_KEY}" + openssl pkey -in "${KBS_KEY}" -pubout -out "${KBS_CERT}" + echo "generate-kbs-keys: wrote ${KBS_KEY} and ${KBS_CERT}" +fi + +if [[ -f "${WORKLOAD_KEY}" ]]; then + echo "generate-kbs-keys: ${WORKLOAD_KEY} already exists, leaving it in place" +else + openssl rand -out "${WORKLOAD_KEY}" 32 + echo "generate-kbs-keys: wrote ${WORKLOAD_KEY}" +fi diff --git a/k8s/coco-dev/scripts/up.sh b/k8s/coco-dev/scripts/up.sh new file mode 100755 index 0000000000..4825065fcc --- /dev/null +++ b/k8s/coco-dev/scripts/up.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Idempotent bring-up for the CoCo no-TEE dev stack (#9213, epic #8534): a local `kind` cluster, the pinned +# confidential-containers Helm chart (dev-only shim override -- see ../helm/values-coco-dev.yaml), and the +# Trustee KBS. Every version/digest below is read from ../versions.json, the single pinned source of truth -- +# see ../README.md for why this topology was chosen and what does/doesn't need real SNP hardware. +# +# Requires: kind, kubectl, helm, jq, openssl, docker. Safe to re-run: each step checks the existing state +# before acting, so re-running after a partial failure resumes rather than duplicating work. +set -euo pipefail + +COCO_DEV_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CLUSTER_NAME="${COCO_DEV_KIND_CLUSTER:-coco-dev}" +VERSIONS_JSON="${COCO_DEV_DIR}/versions.json" + +CHART_REPO="$(jq -r '.cocoHelmChart.repository' "${VERSIONS_JSON}")" +CHART_VERSION="$(jq -r '.cocoHelmChart.version' "${VERSIONS_JSON}")" + +if kind get clusters 2>/dev/null | grep -qx "${CLUSTER_NAME}"; then + echo "up: kind cluster '${CLUSTER_NAME}' already exists" +else + echo "up: creating kind cluster '${CLUSTER_NAME}'" + kind create cluster --name "${CLUSTER_NAME}" +fi + +KUBE_CONTEXT="kind-${CLUSTER_NAME}" + +echo "up: installing confidential-containers chart ${CHART_VERSION} (dev-only shim override)" +helm upgrade --install coco "${CHART_REPO}" \ + --version "${CHART_VERSION}" \ + --namespace coco-system --create-namespace \ + -f "${COCO_DEV_DIR}/helm/values-coco-dev.yaml" \ + --kube-context "${KUBE_CONTEXT}" + +"${COCO_DEV_DIR}/scripts/generate-kbs-keys.sh" + +echo "up: applying KBS manifests" +kubectl apply --context "${KUBE_CONTEXT}" -k "${COCO_DEV_DIR}/kbs/overlays/dev" + +echo "up: waiting for the KBS deployment to become available" +kubectl rollout status deployment/kbs --context "${KUBE_CONTEXT}" -n coco-tenant --timeout=180s + +echo "up: done. RuntimeClasses in the cluster:" +kubectl get runtimeclass --context "${KUBE_CONTEXT}" diff --git a/k8s/coco-dev/versions.json b/k8s/coco-dev/versions.json new file mode 100644 index 0000000000..fa39f1657a --- /dev/null +++ b/k8s/coco-dev/versions.json @@ -0,0 +1,15 @@ +{ + "schemaVersion": 1, + "cocoHelmChart": { + "repository": "oci://ghcr.io/confidential-containers/charts/confidential-containers", + "chart": "confidential-containers", + "version": "0.21.0", + "kataDeploySubchartVersion": "3.31.0", + "devRuntimeClass": "kata-qemu-coco-dev" + }, + "trusteeKbs": { + "image": "ghcr.io/confidential-containers/key-broker-service", + "tag": "built-in-as-v0.21.0", + "digest": "sha256:16926d905621e94dadadb95fd0794dddae8ee30bdf71c71940c6c8fe25087119" + } +} diff --git a/package.json b/package.json index 11c9b95af4..4348f973f6 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "replay-runner-manifest": "tsx scripts/replay-runner-image-manifest.ts", "replay-runner-manifest:write": "tsx scripts/replay-runner-image-manifest.ts --write", "replay-runner-manifest:check": "tsx scripts/replay-runner-image-manifest.ts --check", + "coco-dev-versions:check": "tsx scripts/check-coco-dev-versions.ts", "roadmap:drift-check": "node --experimental-strip-types scripts/check-roadmap-issue-drift.ts", "branding-drift:check": "node --experimental-strip-types scripts/check-branding-drift.ts", "branding-drift:update": "node --experimental-strip-types scripts/check-branding-drift.ts --update", @@ -113,7 +114,7 @@ "test:smoke:browser:install": "playwright install chromium", "test:smoke:browser": "node --experimental-strip-types scripts/smoke-ui-browser.ts", "pretest:ci": "npm run check-node-version", - "test:ci": "git diff --check && npm run actionlint && npm run lint:composite-actions && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run build:mcp && npm run build:miner && npm run build --workspace @loopover/ui-kit && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run test --workspace @loopover/engine && npm run test:workers && npm run test:mcp-pack && npm run test:miner-pack && npm run test:engine-pack && npm run test:ui-kit-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run coverage-boltons:check && npm run import-specifiers:check && npm run replay-runner-manifest:check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run engines-nvmrc:check && npm run release-manifest:sync:check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build", + "test:ci": "git diff --check && npm run actionlint && npm run lint:composite-actions && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run miner:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run build --workspace @loopover/engine && npm run build --workspace @loopover/discovery-index && npm run build:mcp && npm run build:miner && npm run build --workspace @loopover/ui-kit && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:live-gate-parity && npm run test:driver-parity && npm run test --workspace @loopover/engine && npm run test:workers && npm run test:mcp-pack && npm run test:miner-pack && npm run test:engine-pack && npm run test:ui-kit-pack && npm run test:miner-deployment-docs-audit && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run coverage-boltons:check && npm run import-specifiers:check && npm run replay-runner-manifest:check && npm run coco-dev-versions:check && npm run branding-drift:check && npm run manifest:drift-check && npm run engine-parity:drift-check && npm run engines-nvmrc:check && npm run release-manifest:sync:check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build", "test:release": "npm run test:ci && npm run changelog:check", "test:release:mcp": "npm run test:ci", "test:watch": "vitest", diff --git a/scripts/check-coco-dev-versions-core.ts b/scripts/check-coco-dev-versions-core.ts new file mode 100644 index 0000000000..a749cc69f9 --- /dev/null +++ b/scripts/check-coco-dev-versions-core.ts @@ -0,0 +1,59 @@ +// Pure core for the coco-dev version-pin drift check (#9213, epic #8534). k8s/coco-dev/versions.json is the +// single declared source of truth for the Trustee KBS image; kustomize itself needs a literal digest string +// baked into kbs/base/kustomization.yaml (no templating mechanism reaches into it), so that literal is the +// one place a version bump can silently drift from versions.json if a contributor edits one file without the +// other. This module compares the two; it does not check either value against anything upstream -- bumping +// versions.json to a new digest is a deliberate, reviewed change, not something this check second-guesses. +export type CocoDevVersions = { + schemaVersion: number; + trusteeKbs: { + image: string; + tag: string; + digest: string; + }; +}; + +export type KustomizationImage = { + name: string; + newName?: string; + digest?: string; +}; + +export type CocoDevVersionDriftResult = + | { drifted: false } + | { drifted: true; reasons: string[] }; + +/** + * Compare the pinned versions manifest against the image entry actually read out of + * kbs/base/kustomization.yaml. Reports every differing field, not just a single boolean, so a CI failure + * tells a reviewer exactly what to reconcile instead of just that something doesn't match. + */ +export function checkCocoDevVersionDrift(versions: CocoDevVersions, kustomizationImage: KustomizationImage | undefined): CocoDevVersionDriftResult { + const reasons: string[] = []; + + if (!kustomizationImage) { + reasons.push("kbs/base/kustomization.yaml has no images entry named 'kbs-container-image'"); + return { drifted: true, reasons }; + } + + const expectedName = `${versions.trusteeKbs.image}`; + if (kustomizationImage.newName !== expectedName) { + reasons.push(`image name: versions.json declares '${expectedName}', kustomization.yaml has '${kustomizationImage.newName}'`); + } + + if (kustomizationImage.digest !== versions.trusteeKbs.digest) { + reasons.push(`image digest: versions.json declares '${versions.trusteeKbs.digest}', kustomization.yaml has '${kustomizationImage.digest}'`); + } + + return reasons.length > 0 ? { drifted: true, reasons } : { drifted: false }; +} + +/** Find the `images[]` entry kustomize will substitute for the placeholder name `kbs-container-image`, from + * an already-parsed kustomization.yaml document. Returns undefined if absent, letting the caller decide how + * to report that (never throws -- an absent entry is exactly the drift condition being checked for). */ +export function findKustomizationImage(parsedKustomization: unknown, placeholderName: string): KustomizationImage | undefined { + if (typeof parsedKustomization !== "object" || parsedKustomization === null) return undefined; + const images = (parsedKustomization as { images?: unknown }).images; + if (!Array.isArray(images)) return undefined; + return images.find((entry): entry is KustomizationImage => typeof entry === "object" && entry !== null && (entry as { name?: unknown }).name === placeholderName); +} diff --git a/scripts/check-coco-dev-versions.ts b/scripts/check-coco-dev-versions.ts new file mode 100644 index 0000000000..fd1af73c17 --- /dev/null +++ b/scripts/check-coco-dev-versions.ts @@ -0,0 +1,35 @@ +#!/usr/bin/env node +// CLI wrapper for check-coco-dev-versions-core.ts (#9213, epic #8534) -- wired into `npm run test:ci` so a +// version bump made in only one of the two files (k8s/coco-dev/versions.json, +// k8s/coco-dev/kbs/base/kustomization.yaml) fails CI instead of silently shipping a KBS deploy that doesn't +// match its own recorded version manifest. +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +import { parse } from "yaml"; + +import { type CocoDevVersions, checkCocoDevVersionDrift, findKustomizationImage } from "./check-coco-dev-versions-core"; + +const VERSIONS_JSON_PATH = "k8s/coco-dev/versions.json"; +const KUSTOMIZATION_PATH = "k8s/coco-dev/kbs/base/kustomization.yaml"; +const KBS_IMAGE_PLACEHOLDER = "kbs-container-image"; + +function main(): void { + const versions = JSON.parse(readFileSync(VERSIONS_JSON_PATH, "utf8")) as CocoDevVersions; + const kustomization = parse(readFileSync(KUSTOMIZATION_PATH, "utf8")); + const kustomizationImage = findKustomizationImage(kustomization, KBS_IMAGE_PLACEHOLDER); + + const result = checkCocoDevVersionDrift(versions, kustomizationImage); + if (!result.drifted) { + process.stdout.write("coco-dev versions: OK\n"); + return; + } + + process.stderr.write(`coco-dev version drift between ${VERSIONS_JSON_PATH} and ${KUSTOMIZATION_PATH} (#9213):\n`); + for (const reason of result.reasons) { + process.stderr.write(` ${reason}\n`); + } + process.exit(1); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) main(); diff --git a/test/unit/check-coco-dev-versions-core.test.ts b/test/unit/check-coco-dev-versions-core.test.ts new file mode 100644 index 0000000000..3655063eb9 --- /dev/null +++ b/test/unit/check-coco-dev-versions-core.test.ts @@ -0,0 +1,93 @@ +import { describe, expect, it } from "vitest"; + +import { type CocoDevVersions, checkCocoDevVersionDrift, findKustomizationImage } from "../../scripts/check-coco-dev-versions-core"; + +const VERSIONS: CocoDevVersions = { + schemaVersion: 1, + trusteeKbs: { + image: "ghcr.io/confidential-containers/key-broker-service", + tag: "built-in-as-v0.21.0", + digest: "sha256:16926d905621e94dadadb95fd0794dddae8ee30bdf71c71940c6c8fe25087119", + }, +}; + +describe("checkCocoDevVersionDrift", () => { + it("reports no drift when the kustomization image matches versions.json exactly", () => { + const result = checkCocoDevVersionDrift(VERSIONS, { + name: "kbs-container-image", + newName: "ghcr.io/confidential-containers/key-broker-service", + digest: "sha256:16926d905621e94dadadb95fd0794dddae8ee30bdf71c71940c6c8fe25087119", + }); + expect(result).toEqual({ drifted: false }); + }); + + it("reports drift when the digest was bumped in versions.json but not in kustomization.yaml", () => { + const result = checkCocoDevVersionDrift(VERSIONS, { + name: "kbs-container-image", + newName: "ghcr.io/confidential-containers/key-broker-service", + digest: "sha256:0000000000000000000000000000000000000000000000000000000000000000", + }); + expect(result.drifted).toBe(true); + expect(result.drifted && result.reasons.join("\n")).toMatch(/image digest/); + }); + + it("reports drift when the image name diverges", () => { + const result = checkCocoDevVersionDrift(VERSIONS, { + name: "kbs-container-image", + newName: "ghcr.io/some-fork/key-broker-service", + digest: "sha256:16926d905621e94dadadb95fd0794dddae8ee30bdf71c71940c6c8fe25087119", + }); + expect(result.drifted).toBe(true); + expect(result.drifted && result.reasons.join("\n")).toMatch(/image name/); + }); + + it("reports both reasons when both name and digest diverge", () => { + const result = checkCocoDevVersionDrift(VERSIONS, { + name: "kbs-container-image", + newName: "ghcr.io/some-fork/key-broker-service", + digest: "sha256:0000000000000000000000000000000000000000000000000000000000000000", + }); + expect(result.drifted).toBe(true); + expect(result.drifted && result.reasons).toHaveLength(2); + }); + + it("reports drift when the kustomization has no matching images entry at all", () => { + const result = checkCocoDevVersionDrift(VERSIONS, undefined); + expect(result.drifted).toBe(true); + expect(result.drifted && result.reasons.join("\n")).toMatch(/no images entry/); + }); +}); + +describe("findKustomizationImage", () => { + it("finds the matching entry by placeholder name among several images", () => { + const parsed = { + images: [ + { name: "some-other-image", newName: "example.com/other" }, + { name: "kbs-container-image", newName: "ghcr.io/confidential-containers/key-broker-service", digest: "sha256:abc" }, + ], + }; + expect(findKustomizationImage(parsed, "kbs-container-image")).toEqual({ + name: "kbs-container-image", + newName: "ghcr.io/confidential-containers/key-broker-service", + digest: "sha256:abc", + }); + }); + + it("returns undefined when no images array is present", () => { + expect(findKustomizationImage({ resources: ["namespace.yaml"] }, "kbs-container-image")).toBeUndefined(); + }); + + it("returns undefined when images is present but not an array", () => { + expect(findKustomizationImage({ images: "not-an-array" }, "kbs-container-image")).toBeUndefined(); + }); + + it("returns undefined when the parsed document is not an object", () => { + expect(findKustomizationImage(null, "kbs-container-image")).toBeUndefined(); + expect(findKustomizationImage("a string", "kbs-container-image")).toBeUndefined(); + }); + + it("returns undefined when no entry in images matches the placeholder name", () => { + const parsed = { images: [{ name: "unrelated-image" }] }; + expect(findKustomizationImage(parsed, "kbs-container-image")).toBeUndefined(); + }); +});