From b9cf1fac4fd0cfbf64f87d26e7dbd6065316c157 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Fri, 10 Jul 2026 21:49:07 +0200 Subject: [PATCH 1/2] refactor(husk): restore and resume the guest while dormant, not while a tenant waits Slice 2 of docs/superpowers/plans/2026-07-10-prepare-time-restore.md, building on the tap-at-prepare slice (#883). Together they move almost the whole warm-claim activate off the hot path. Measured on prod, one matched claim: the husk activate is 112.8 ms, of which the restore (vmstate_restore 22.9), the resume (2.4), and the guest-ready wait (40.6, largely demand fault-in under the lazy UFFD restore) are ~66 ms. All of it is a function of the pod and its template, not of the claim. Only the fork-correctness handshake (fresh entropy, the clock step, the per-VM network re-addressing, and the tenant secrets) and serving the sandbox API with the claim token actually need the claim. Behind --husk-prepare-restore (which requires --husk-prepare-egress-link, because Firecracker binds the baked NIC to the tap at snapshot load), a warm pod now loads its default VM's snapshot and RESUMES its guest while dormant. Activate takes a fast path when the instance is pre-restored: it skips the restore, the rootfs rebind, the resume, and the guest-ready wait, re-dials the already-running guest (sub-millisecond, the vsock listener has been up since the prepare-time resume), and pays only the handshake. The handshake-and-serve tail is extracted into activateHandshakeAndServe so both paths share it byte-for-byte. Fail-closed, which is the load-bearing part: - Prepare refuses to go dormant if the restore, rootfs rebind, resume, or guest-ready fails, so the pool never offers a half-restored pod. - Activate refuses a claim whose snapshot dir differs from the one pre-restored, with a named error, because a resumed guest cannot be reloaded and must never be served against the wrong image. - Only the DEFAULT VM is pre-restored. A co-located fork child (its own snapshot, childUFFD backend, per-fork tap) and the pre-warm generic child are excluded, so the fork paths are unchanged. - The ready conn proven at prepare is CLOSED, not held across dormancy where it could go stale. The dormant guest is a new surface and the threat model records it: it serves no tenant (the sandbox API is bound and the token delivered only at the claim), it is reseeded fail-closed at the claim's NotifyForked exactly as a restore-at-activate guest is (the dormancy window only enlarges the absolute clock step), and it holds its working set resident (~72 MiB/sandbox) versus ~0 for a never-loaded VMM. DEFAULT OFF at every layer: cmd/controller rejects --husk-prepare-restore without --husk-prepare-egress-link (and therefore without --multi-vm-fork), the pod builder emits --prepare-restore only inside the prepared-link block, and the stub no-ops unless the tap was prepared. Off, the restore path is byte-for-byte the activate-time one. NOT KVM-verified and NOT canaried on prod: this needs a firecracker-test gate and a one-pod canary before it may be called verified, and it is the highest-risk change in the latency effort (it resumes a live guest during dormancy on the tenant-VM boot path). Verified with the mock VMM (the state machine is engine-independent): TestPrepareRestoresAndResumesTheGuestWhileDormant (load+resume at prepare, none at activate), TestPrepareRestoreIsOptIn (default off is byte-for-byte activate-time), TestActivateFailsClosedOnASnapshotMismatchAfterPrepareRestore, TestBuildHuskPodThreadsPrepareRestore. Each fails on the unfixed code. go test -race ./internal/husk/, go test ./internal/controller/ ./cmd/husk-stub/, and golangci-lint clean for both GOOS=darwin and GOOS=linux. internal/husk is a security-sensitive path per CLAUDE.md and needs a named human reviewer. Signed-off-by: Jannes Stubbemann --- cmd/controller/main.go | 17 ++- cmd/controller/main_test.go | 20 +-- cmd/husk-stub/main.go | 2 + .../plans/2026-07-10-prepare-time-restore.md | 17 ++- docs/threat-model.md | 2 +- internal/controller/huskpod.go | 11 ++ internal/controller/huskpod_test.go | 29 +++++ .../controller/sandboxclaim_controller.go | 3 + internal/controller/sandboxpool_controller.go | 3 + internal/husk/multivm.go | 113 ++++++++++++++++- internal/husk/multivm_network_test.go | 119 ++++++++++++++++++ internal/husk/stub.go | 20 ++- 12 files changed, 337 insertions(+), 19 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 3be74a62..9e220af7 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -46,11 +46,18 @@ func init() { // // --husk-prepare-egress-link needs BOTH --multi-vm-fork (the stub prepares its tap only // on the multi-VM Prepare path) AND the husk-pods run path (the raw-forkd path never -// creates a husk pod, so there is no pod to prepare a tap in). -func validatePrepareFlags(prepareEgressLink, multiVMFork, enableHuskPods bool) error { +// creates a husk pod, so there is no pod to prepare a tap in). --husk-prepare-restore +// needs --husk-prepare-egress-link, and therefore everything that needs. +func validatePrepareFlags(prepareEgressLink, prepareRestore, multiVMFork, enableHuskPods bool) error { if prepareEgressLink && (!multiVMFork || !enableHuskPods) { return fmt.Errorf("--husk-prepare-egress-link requires --multi-vm-fork AND the husk-pods run path (not --enable-raw-forkd or --mock): the tap is prepared only by a multi-VM husk pod, which the raw-forkd path never creates") } + // Prepare-time restore builds ON the prepared tap (Firecracker binds the baked NIC + // to it at snapshot load), so it requires the egress link, and transitively + // everything the egress link requires. + if prepareRestore && !prepareEgressLink { + return fmt.Errorf("--husk-prepare-restore requires --husk-prepare-egress-link: the tap must exist before the snapshot load") + } return nil } @@ -81,6 +88,7 @@ func main() { var liveCowChildImport bool var prewarmChild bool var prepareEgressLink bool + var prepareRestore bool var huskStubImage string var huskDNSUpstream string var huskControlPort int @@ -113,6 +121,7 @@ func main() { flag.BoolVar(&multiVMFork, "multi-vm-fork", false, "EXPERIMENTAL, DEFAULT OFF: route a hosted fork child into an ADDITIONAL VM spawned INSIDE the source husk pod (the spawn-vm control op) instead of a brand-new child pod, when the source pod is multi-VM capable (its stub runs --multi-vm). OFF is byte-for-byte the current new-pod-per-fork path, so nothing changes unless you opt in AND the source pod is multi-VM capable; a non-capable source silently falls back to a new pod. This wires the routing + status (child status.pod = source pod, status.vmId = the spawned VM); the per-pod and node memory accounting that pends or spills a fork when a pod is full is a follow-up, so co-location is capped conservatively and the remainder spills to new pods. Only used with --enable-husk-pods.") flag.BoolVar(&liveCowFork, "live-cow-fork", false, "EXPERIMENTAL, DEFAULT OFF: start warm husk pods with --live-cow-fork so a CO-LOCATED fork child shares the PARENT's resident guest memory (patched Firecracker memfd + userfaultfd write-protect) instead of restoring from the disk fork snapshot (milestone m4b), driving the hosted fork toward sub-100ms. SEPARATE from --multi-vm-fork so it can be deployed off and canaried independently. OFF is byte-for-byte the current disk co-location. The co-located child still falls back to the disk restore where the child-side memfd import is not yet complete, so turning it on never breaks a fork; it is a no-op off Linux (userfaultfd write-protect is Linux-only). Only meaningful with --enable-husk-pods.") flag.BoolVar(&liveCowChildImport, "live-cow-child-import", false, "EXPERIMENTAL, DEFAULT OFF: with --live-cow-fork on, opt warm husk pods onto the VMSTATE-ONLY fork capture (skip the ~364ms disk mem write) so a co-located child boots its guest RAM from the source shared memfd instead of the disk fork snapshot. REQUIRES a child-side-import patched Firecracker; off keeps the armed source writing the disk mem so every child restores from disk and no fork hangs. Only meaningful with --live-cow-fork and --enable-husk-pods.") + flag.BoolVar(&prepareRestore, "husk-prepare-restore", false, "EXPERIMENTAL, DEFAULT OFF: warm husk pods load their default VM's snapshot and RESUME its guest while DORMANT, so a claim pays only the fork-correctness handshake instead of the restore, resume, and guest-ready wait. REQUIRES --husk-prepare-egress-link (the tap must exist before the snapshot load) and --multi-vm-fork. Off keeps the whole restore on the activate path byte-for-byte.") flag.BoolVar(&prepareEgressLink, "husk-prepare-egress-link", false, "EXPERIMENTAL, DEFAULT OFF: warm husk pods bring their default VM's tap up while DORMANT, under a default-deny policy, so a claim pays only the atomic nft transaction that installs the tenant's policy instead of also paying the tap create. Requires --multi-vm-fork and --enable-husk-pods. Off keeps the whole filter on the activate path byte-for-byte.") flag.BoolVar(&prewarmChild, "prewarm-child", false, "EXPERIMENTAL, DEFAULT OFF: keep one dormant generic co-located child Firecracker pre-prepared per multi-vm husk pod so a fork adopts the ready child (fc_boot=0, prepare off the hot path) instead of booting one at fork time. Requires --multi-vm-fork and --enable-husk-pods; a fresh slot re-warms async, a miss falls back to on-demand prepare byte-for-byte.") flag.BoolVar(&huskConnReuse, "husk-conn-reuse", false, "EXPERIMENTAL, DEFAULT OFF: reuse ONE authenticated mTLS husk control connection per husk pod across control-plane RPCs (activate, fork-snapshot, spawn-vm, remove-fork-snapshot) instead of opening a fresh TCP+TLS handshake per RPC. A co-located fork does fork-snapshot then spawn-vm to the SAME source pod, so reuse saves the second full handshake and cuts the per-RPC connection-setup overhead toward zero, driving the hosted fork toward sub-100ms. OFF is byte-for-byte the current one-shot dial-per-RPC path. The husk server always supports both (a one-shot client that closes after one request and a reused connection that sends several), so this flag only changes the CONTROLLER side and can be canaried + rolled back independently. mTLS identity is verified on every dial and one request is in flight per connection, so a reused connection is neither less authenticated nor frame-interleaved (see docs/threat-model.md). Only meaningful with --enable-husk-pods.") @@ -161,7 +170,7 @@ func main() { // without --multi-vm-fork would hand the pool builder MultiVM=false while the claim // reconciler believed the link was prepared. An operator who asked for a latency // optimization must not be left with a pod shape that cannot deliver it. - if err := validatePrepareFlags(prepareEgressLink, multiVMFork, enableHuskPods); err != nil { + if err := validatePrepareFlags(prepareEgressLink, prepareRestore, multiVMFork, enableHuskPods); err != nil { logger.Error(err, "invalid prepare-time flags") os.Exit(1) } @@ -263,6 +272,7 @@ func main() { LiveCowChildImport: liveCowChildImport, PrewarmChild: prewarmChild, PrepareEgressLink: prepareEgressLink, + PrepareRestore: prepareRestore, HuskStubImage: huskStubImage, HuskDNSUpstream: huskDNSUpstream, KVMResourceName: "mitos.run/kvm", @@ -309,6 +319,7 @@ func main() { LiveCowChildImport: liveCowChildImport, PrewarmChild: prewarmChild, PrepareEgressLink: prepareEgressLink, + PrepareRestore: prepareRestore, HuskControlPort: huskControlPort, HuskStubImage: huskStubImage, HuskDNSUpstream: huskDNSUpstream, diff --git a/cmd/controller/main_test.go b/cmd/controller/main_test.go index 50c5bb43..27a69ea4 100644 --- a/cmd/controller/main_test.go +++ b/cmd/controller/main_test.go @@ -36,20 +36,22 @@ func TestResolveRunMode(t *testing.T) { func TestValidatePrepareFlags(t *testing.T) { cases := []struct { - name string - egress, multiVM, huskPods, wantErr bool + name string + egress, restore, multiVM, huskPods, wantErr bool }{ - {"off is fine", false, false, false, false}, - {"egress with multivm and husk pods", true, true, true, false}, - {"egress without multivm", true, false, true, true}, - {"egress without husk pods (raw-forkd path)", true, true, false, true}, - {"egress on raw-forkd and single-vm", true, false, false, true}, + {"off is fine", false, false, false, false, false}, + {"egress with multivm and husk pods", true, false, true, true, false}, + {"egress without multivm", true, false, false, true, true}, + {"egress without husk pods (raw-forkd path)", true, false, true, false, true}, + {"restore with egress, multivm, husk pods", true, true, true, true, false}, + {"restore without egress", false, true, true, true, true}, + {"restore with egress but no multivm", true, true, false, true, true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - err := validatePrepareFlags(tc.egress, tc.multiVM, tc.huskPods) + err := validatePrepareFlags(tc.egress, tc.restore, tc.multiVM, tc.huskPods) if (err != nil) != tc.wantErr { - t.Errorf("validatePrepareFlags(%v,%v,%v) err=%v, wantErr=%v", tc.egress, tc.multiVM, tc.huskPods, err, tc.wantErr) + t.Errorf("validatePrepareFlags(egress=%v,restore=%v,multiVM=%v,huskPods=%v) err=%v, wantErr=%v", tc.egress, tc.restore, tc.multiVM, tc.huskPods, err, tc.wantErr) } }) } diff --git a/cmd/husk-stub/main.go b/cmd/husk-stub/main.go index 945b0b6f..6b977b63 100644 --- a/cmd/husk-stub/main.go +++ b/cmd/husk-stub/main.go @@ -196,6 +196,7 @@ func run() error { prepareEgressLink = flag.Bool("prepare-egress-link", false, "EXPERIMENTAL, DEFAULT OFF: bring the pod's default-VM tap up while the pod is DORMANT, under a default-deny policy, so a claim pays only the atomic nft transaction that installs the tenant's policy instead of also paying the tap create (measured: the link half is about two thirds of the egress_filter stage). Requires --multi-vm and --in-pod-guest-ip. Off keeps the whole filter on the activate path byte-for-byte. Not a secret.") inPodGuestIP = flag.String("in-pod-guest-ip", "", "the fixed in-pod guest IP of this pod's default VM (the same value the controller sends in the activate request). Needed BEFORE that request arrives by --prepare-egress-link, because the tap name derives from it. Config, not a secret.") inPodGatewayIP = flag.String("in-pod-gateway-ip", "", "the fixed in-pod gateway IP paired with --in-pod-guest-ip. Config, not a secret.") + prepareRestore = flag.Bool("prepare-restore", false, "EXPERIMENTAL, DEFAULT OFF: load the default VM's snapshot and RESUME its guest while the pod is DORMANT, so a claim pays only the fork-correctness handshake instead of the restore, resume, and guest-ready wait. REQUIRES --prepare-egress-link (the tap must exist before the snapshot load). Off keeps the whole restore on the activate path byte-for-byte. Not a secret.") prewarmChild = flag.Bool("prewarm-child", false, "EXPERIMENTAL, DEFAULT OFF: keep ONE dormant generic co-located child Firecracker pre-prepared per multi-vm pod so a fork ADOPTS the ready child (fc_boot=0, prepare off the hot path) instead of booting one at fork time. Requires --multi-vm; a fresh slot re-warms async, a miss falls back to the on-demand prepare. Not a secret.") liveCowChildImport = flag.Bool("live-cow-child-import", false, "EXPERIMENTAL, default false: opt the live-cow fork onto the VMSTATE-ONLY capture (skip the ~364ms disk mem write). REQUIRES a shipped child-side memfd-import Firecracker patch: the co-located child boots its guest RAM from the source shared memfd instead of the disk fork snapshot. The currently shipped patched binary patches the SOURCE (restore) side only, so leave this OFF; with it off an armed --live-cow-fork source still writes the disk mem, so every co-located child restores from disk and the fork never hangs. Turn on ONLY once a child-side import binary is shipped. Not a secret.") ) @@ -441,6 +442,7 @@ func run() error { LiveCowChildImport: *liveCowChildImport, PrewarmChild: *prewarmChild, PrepareEgressLink: *prepareEgressLink, + PrepareRestore: *prepareRestore, InPodGuestIP: *inPodGuestIP, InPodGatewayIP: *inPodGatewayIP, }) diff --git a/docs/superpowers/plans/2026-07-10-prepare-time-restore.md b/docs/superpowers/plans/2026-07-10-prepare-time-restore.md index 2e181ddf..b71fb3a8 100644 --- a/docs/superpowers/plans/2026-07-10-prepare-time-restore.md +++ b/docs/superpowers/plans/2026-07-10-prepare-time-restore.md @@ -1,6 +1,6 @@ # Prepare-time restore: move the snapshot restore off the warm-claim hot path -Status: slice 1 implemented (default off, not yet canaried on prod); slices 2 and 3 designed. +Status: slices 1 and 2 implemented (default off, not yet canaried on prod); slice 3 designed. ## Why @@ -123,9 +123,18 @@ Expected activate: roughly `nft` + `handshake` + `serve_api`, i.e. ~30 ms agains Requires `--multi-vm-fork`; the controller refuses the combination at startup and the pod builder refuses it again. Still to do: measure it on prod behind a one-pod canary, then a KVM gate. Until both, the threat model does NOT call this path verified. -2. **Restore at Prepare.** Move `setLiveCowMemSource` / `LoadSnapshot` / `PatchDrive` / - `Resume` / guest-ready into `prepareInstance`, behind the same flag, with the guards - above. Activate becomes policy + handshake + serve_api. +2. **Restore at Prepare.** DONE, default off. Behind `--husk-prepare-restore` (which + requires `--husk-prepare-egress-link`), `prepareRestoreDefaultVM` runs + `setLiveCowMemSource` / `LoadSnapshotWithOverrides` / `PatchDrive` / `Resume` / + guest-ready for the DEFAULT VM after the tap is up, closing the ready conn rather than + holding it across dormancy. Activate takes a fast path when `inst.preRestored`: it + skips the restore, re-dials the already-running guest, and pays only the handshake via + the shared `activateHandshakeAndServe`. FAIL CLOSED on a snapshot-dir mismatch (a + resumed guest cannot be reloaded). Co-located fork children and the pre-warm child are + never pre-restored. Pinned by `TestPrepareRestoresAndResumesTheGuestWhileDormant`, + `TestPrepareRestoreIsOptIn`, `TestActivateFailsClosedOnASnapshotMismatchAfterPrepareRestore`, + `TestBuildHuskPodThreadsPrepareRestore`. Still to do: a KVM gate and a one-pod prod + canary before it is called verified. 3. **Prefault the kernel.** Run the inert warm cell at Prepare so the first tenant `run_code` does not fault the ipykernel's pages in. diff --git a/docs/threat-model.md b/docs/threat-model.md index 028d9f4a..023bc225 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -1219,7 +1219,7 @@ VMMs run jailed under throwaway uids. | Husk pod default ServiceAccount token automount | **mitigated (shipped)** | The husk pod spec now sets `AutomountServiceAccountToken: false` (`internal/controller/huskpod.go` `buildHuskPod`), so the kubelet does NOT mount the namespace default ServiceAccount token into a husk pod. The stub speaks vsock + mTLS and never calls the Kubernetes API (verified: no client-go, no `InClusterConfig`, no SA token read in `cmd/husk-stub` or `internal/husk`), so the token was dead weight; a guest that escapes into the stub no longer inherits a free `system:authenticated` token. The opt-out applies to BOTH warm pods and fork-child pods, which share the builder, proven in `TestBuildHuskPodDisablesSATokenAutomount`. | | forkd gRPC fails OPEN without TLS flags | **mitigated (shipped)** | `grpcServerOptions` (`cmd/forkd/main.go`) now FAILS CLOSED: with no TLS flags it returns a fatal error naming the missing `--tls-cert/--tls-key/--tls-ca` flags and the opt-in, so forkd refuses to start unauthenticated. The legacy insecure-with-loud-warning behavior is reachable only behind an explicit `--allow-insecure-grpc` opt-in (default false) for local development. A partial TLS triple is still a configuration error. The shipped DaemonSet always sets TLS, so production is unaffected; this only stops a silent-insecure misconfig. Proven in `TestGRPCServerOptionsFailClosed` (refuses with no TLS and no opt-in; allowed with the opt-in; allowed with a full TLS triple; partial is an error). | | Host-side vsock read has no deadline | **mitigated (shipped)** | The host-side `vsock.Client` now applies a per-request read deadline in `send` (`internal/vsock/client.go`), defaulting to `DefaultRequestTimeout` (60s) and overridable via `SetRequestTimeout`. The CONNECT preamble reads in `Connect`/`DialStream`/`DialStreamUnix` are likewise bounded (then cleared so the long-lived streaming reads stay ctx/`conn.Close`-cancelled). A malicious or wedged guest that connects then stalls (or dribbles a partial line under the `MaxMessageBytes` cap) now causes the one-shot call to return a timeout error rather than hang the host caller goroutine, vsock fd, and (for the husk stub) stream slot indefinitely. Proven in `TestSendReadDeadlineUnblocksOnStall` (a fake agent that connects then never responds makes `Ping` return rather than hang). | -| Husk egress enforcement | **mitigated (IMPLEMENTED and KVM-VERIFIED end to end)** | See section 0 surface 5 and section 4. The husk default path enforces an in-pod nftables default-deny egress filter in the pod's own netns (CNI-independent, the guarantee), an unconditional cloud-metadata block (`169.254.169.254`, `169.254.0.0/16`, IPv6 `fd00:ec2::254`, before any allow), and the threaded per-template allowlist with a per-pod DNS proxy (failover upstream list, recommended `1.1.1.1:53,8.8.8.8:53`, not cluster DNS); a best-effort controller-emitted NetworkPolicy adds defense in depth (CNI-dependent, cannot express name-based allows). Costs one scoped `NET_ADMIN` capability (ADR 0006) plus, on name-egress pools only, a short-lived privileged `enable-ip-forward` init container that sets `net.ipv4.ip_forward=1` in the pod netns and exits (one-shot; no node prerequisite; the long-lived husk-stub container stays unprivileged). VERIFIED: the husk-network KVM cluster e2e (`test/cluster-e2e/husk-network-e2e.sh`, `cluster-husk-network-e2e`) PASSES on the Hetzner Talos KVM cluster; the claim reaches Ready and all three in-VM enforcement assertions are green inside a real restored VM (metadata-blocked exit 1, default-deny exit 1, allowlist-works `example.com` connect exit 0), verified twice (with and without a node sysctl allowance), proving NO node prerequisite. PREPARE-TIME LINK (`--husk-prepare-egress-link`, controller; `--prepare-egress-link`, stub; EXPERIMENTAL, DEFAULT OFF, and **NOT covered by the KVM verification above**: that verification exercised the activation-time filter, and the prepare-time path still needs a production canary and its own KVM gate before it may be called verified; it also requires `--multi-vm-fork`, which the controller enforces at startup and the pod builder enforces again): a warm multi-VM husk pod may bring its default VM's tap up while it is DORMANT so a claim pays only the nft transaction, not the tap create. The dormant tap is created together with a DEFAULT-DENY policy in the same Prepare, and Prepare FAILS CLOSED (the pod never reaches dormant, so the pool never offers it) if either half fails, so a tap never exists without a policy. The claim REPLACES that policy with the tenant's in ONE atomic `nft -f -` transaction, so there is no window in which a VM is unfiltered, and a rejected policy tears the tap down and clears the prepared marker so the retry rebuilds the link rather than installing a policy on a tap that no longer exists. Nothing is loaded behind a dormant tap today (the snapshot restore stays on the activate path), and the pod netns is not shared with any tenant. Off, the filter path is byte-for-byte the activate-time one. | +| Husk egress enforcement | **mitigated (IMPLEMENTED and KVM-VERIFIED end to end)** | See section 0 surface 5 and section 4. The husk default path enforces an in-pod nftables default-deny egress filter in the pod's own netns (CNI-independent, the guarantee), an unconditional cloud-metadata block (`169.254.169.254`, `169.254.0.0/16`, IPv6 `fd00:ec2::254`, before any allow), and the threaded per-template allowlist with a per-pod DNS proxy (failover upstream list, recommended `1.1.1.1:53,8.8.8.8:53`, not cluster DNS); a best-effort controller-emitted NetworkPolicy adds defense in depth (CNI-dependent, cannot express name-based allows). Costs one scoped `NET_ADMIN` capability (ADR 0006) plus, on name-egress pools only, a short-lived privileged `enable-ip-forward` init container that sets `net.ipv4.ip_forward=1` in the pod netns and exits (one-shot; no node prerequisite; the long-lived husk-stub container stays unprivileged). VERIFIED: the husk-network KVM cluster e2e (`test/cluster-e2e/husk-network-e2e.sh`, `cluster-husk-network-e2e`) PASSES on the Hetzner Talos KVM cluster; the claim reaches Ready and all three in-VM enforcement assertions are green inside a real restored VM (metadata-blocked exit 1, default-deny exit 1, allowlist-works `example.com` connect exit 0), verified twice (with and without a node sysctl allowance), proving NO node prerequisite. PREPARE-TIME LINK (`--husk-prepare-egress-link`, controller; `--prepare-egress-link`, stub; EXPERIMENTAL, DEFAULT OFF, and **NOT covered by the KVM verification above**: that verification exercised the activation-time filter, and the prepare-time path still needs a production canary and its own KVM gate before it may be called verified; it also requires `--multi-vm-fork`, which the controller enforces at startup and the pod builder enforces again): a warm multi-VM husk pod may bring its default VM's tap up while it is DORMANT so a claim pays only the nft transaction, not the tap create. The dormant tap is created together with a DEFAULT-DENY policy in the same Prepare, and Prepare FAILS CLOSED (the pod never reaches dormant, so the pool never offers it) if either half fails, so a tap never exists without a policy. The claim REPLACES that policy with the tenant's in ONE atomic `nft -f -` transaction, so there is no window in which a VM is unfiltered, and a rejected policy tears the tap down and clears the prepared marker so the retry rebuilds the link rather than installing a policy on a tap that no longer exists. Nothing is loaded behind a dormant tap today (unless prepare-time restore is also on, below), and the pod netns is not shared with any tenant. Off, the filter path is byte-for-byte the activate-time one. PREPARE-TIME RESTORE (`--husk-prepare-restore`, controller / stub; EXPERIMENTAL, DEFAULT OFF, REQUIRES `--husk-prepare-egress-link`, NOT KVM-verified) goes further: the pod loads its default VM's snapshot and RESUMES its guest while dormant, so a live guest runs behind the default-deny prepared tap before any tenant is attached. It serves NO tenant during dormancy (the sandbox API is bound only at the claim's serve_api step and no token is delivered until the claim's handshake), it is reseeded FAIL-CLOSED at the claim's NotifyForked exactly as a restore-at-activate guest is (the dormancy window only enlarges the absolute clock step), and activate FAILS CLOSED if the claim names a different snapshot than the one pre-restored, because a resumed guest cannot be reloaded. Cost: a pre-restored dormant VM holds its working set resident (~72 MiB/sandbox on the lazy path) versus ~0 for a never-loaded VMM. Off, the restore path is byte-for-byte the activate-time one. | | agents.x-k8s.io facade admission surface (v1beta1 conformance, #357) | **mitigated (bounded translation, no host surface)** | The conformance facade (`internal/facade`, opt-in deployment) watches tenant-authored upstream `agents.x-k8s.io/v1beta1` and `extensions.agents.x-k8s.io/v1beta1` custom resources (Sandbox, SandboxTemplate, SandboxWarmPool, SandboxClaim) and TRANSLATES each into our `mitos.run/v1` SandboxPool/Sandbox objects in the SAME namespace, owner-referenced to the upstream CR for GC. It creates only namespace-scoped objects and grants no cross-namespace or host access; the run-path security boundary is unchanged (the bridged Sandbox is governed by the husk-pod and sandbox-API controls above, not by the facade). Tenant-controlled upstream fields are BOUNDED: `spec.operatingMode` (Running/Suspended) drives only release/re-activate of the bridged claim; `SandboxClaim.spec.warmPoolRef.name` is a name lookup of OUR pool of that name (no path or host input); `spec.podTemplate.spec.containers[0].env` (including `env.valueFrom` references) is mirrored through by reference and never logged; all other podTemplate fields and `volumeClaimTemplates` are UNMAPPED (no tenant-driven volume provisioning or host-shaped field reaches our engine via the facade, see docs/facade-conformance.md exceptions 1, 3, 4, 5). The v0.5.0 migration to v1beta1 REDUCED the input surface: it removed the v1alpha1 `warmpool` policy resolution branch (none/default/named), so a claim now references a warm pool by name with no policy logic to abuse. Test-infra note: v1beta1 is the storage version with a `conversion.strategy: Webhook` we do not run; the kind conformance job pins the live CRD conversion to None at install time and exercises only the storage version (the vendored CRD on disk is unchanged). Residual: the facade trusts cluster RBAC that admits the upstream CRs, so whoever may create an `agents.x-k8s.io` Sandbox in a namespace can drive a pool binding there; the same-namespace tenancy boundary (section 7a) applies, and the facade adds no privilege beyond it. | | Control-plane egress lockdown (chart profile, issue #704) | **mitigated (opt-in)** | The chart ships an opt-in default-deny EGRESS profile for the control plane (`networkPolicy.controlPlane.enabled`, `templates/control-plane-netpol.yaml`): a CiliumNetworkPolicy selecting the controller, console, and gateway that enumerates every legitimate flow (DNS via kube-dns with L7 visibility, the kube-apiserver/host/remote-node entities, intra-namespace) and denies the rest, bounding what a COMPROMISED control-plane component can reach (no lateral movement into other namespaces, no arbitrary internet egress for exfiltration). When `console.onboarding.smtp.host` is set, a SECOND policy scoped to the console ALONE additionally allows that SMTP host and port; it is DERIVED from the same values that configure the sender, so the network allow cannot drift from the sender config (the drift class that silently broke hosted signups: SMTP env present, egress absent, every verification send timed out). The SMTP policy never renders without the baseline profile (a lone egress policy would itself flip the console to default-deny), and the controller and gateway gain no mail-relay egress (a compromised controller cannot exfiltrate over SMTP). `extraEgress` passes deployment-specific rules through verbatim (e.g. an in-cluster OIDC issuer namespace); rules added there widen the boundary and are the operator's responsibility. Requires the Cilium CNI (CiliumNetworkPolicy CRD), consistent with the NetworkPolicy-enforcing-CNI requirement stated for the tenancy boundary. Default OFF: a plain install is unchanged. Rendering is asserted in `deploy/charts/charttest/netpol_test.go` (opt-in absent by default, baseline flows present, SMTP allow derived and console-scoped, all-or-nothing coupling). Residual: this is an egress control only (ingress lockdown for the control plane is separate work), and on a non-Cilium CNI the profile cannot be expressed (values accepted but objects unusable; the CRD requirement is documented). | diff --git a/internal/controller/huskpod.go b/internal/controller/huskpod.go index 48466d71..14406d18 100644 --- a/internal/controller/huskpod.go +++ b/internal/controller/huskpod.go @@ -205,6 +205,10 @@ type HuskPodOptions struct { // pod brings its tap up while dormant and a claim pays only the nft transaction // that installs the tenant's policy. Requires MultiVM. Default off. PrepareEgressLink bool + // PrepareRestore starts the stub with --prepare-restore so a warm pod loads its + // default VM's snapshot and resumes its guest while dormant, leaving only the + // handshake on the warm-claim hot path. REQUIRES PrepareEgressLink (and MultiVM). + PrepareRestore bool // MultiVMForkVMs is the number of ADDITIONAL fork-child VMs a multi-VM pod // reserves node memory for up front (beyond the source VM), so the co-location // routing has room to co-locate that many children before a fork spills to a new @@ -581,6 +585,12 @@ func (r *SandboxPoolReconciler) buildHuskPod(pool *v1.SandboxPool, template *v1. "--in-pod-guest-ip", huskGuestIP, "--in-pod-gateway-ip", huskGatewayIP, ) + // Prepare-time restore builds ON the prepared tap, so it is emitted only inside + // this block: --prepare-restore without --prepare-egress-link would have no tap + // at snapshot load. The stub also refuses that combination. + if opts.PrepareRestore { + args = append(args, "--prepare-restore") + } } // Live-cow write-protect needs a KERNEL-MODE userfaultfd over the guest RAM: the @@ -1416,6 +1426,7 @@ func (r *SandboxPoolReconciler) reconcileHuskPods(ctx context.Context, pool *v1. LiveCowChildImport: r.LiveCowChildImport, PrewarmChild: r.PrewarmChild, PrepareEgressLink: r.PrepareEgressLink, + PrepareRestore: r.PrepareRestore, MultiVMForkVMs: r.MultiVMForkVMs, StubImage: r.HuskStubImage, DNSUpstream: r.HuskDNSUpstream, diff --git a/internal/controller/huskpod_test.go b/internal/controller/huskpod_test.go index 1d884b4d..17d4d1bf 100644 --- a/internal/controller/huskpod_test.go +++ b/internal/controller/huskpod_test.go @@ -1556,3 +1556,32 @@ func TestBuildHuskPodOmitsPrepareEgressLinkWithoutMultiVM(t *testing.T) { } } } + +// TestBuildHuskPodThreadsPrepareRestore proves --prepare-restore flows to the stub ONLY +// alongside --prepare-egress-link (and therefore MultiVM), because the restore builds on +// the prepared tap. Off, the flag is absent. +func TestBuildHuskPodThreadsPrepareRestore(t *testing.T) { + r := &controller.SandboxPoolReconciler{} + pool := &v1.SandboxPool{ObjectMeta: metav1.ObjectMeta{Name: "p", Namespace: "ns"}} + tmpl := &v1.PoolTemplateSpec{} + + on := r.BuildHuskPodForTest(pool, tmpl, controller.HuskPodOptions{ + StubImage: "img", MultiVM: true, PrepareEgressLink: true, PrepareRestore: true, + }) + if !argsContain(on.Spec.Containers[0].Args, "--prepare-restore") { + t.Errorf("husk args missing --prepare-restore when enabled: %v", on.Spec.Containers[0].Args) + } + + // Without the prepared link, restore has no tap; the flag must not be emitted. + noLink := r.BuildHuskPodForTest(pool, tmpl, controller.HuskPodOptions{ + StubImage: "img", MultiVM: true, PrepareRestore: true, + }) + if argsContain(noLink.Spec.Containers[0].Args, "--prepare-restore") { + t.Errorf("husk args carry --prepare-restore without --prepare-egress-link: %v", noLink.Spec.Containers[0].Args) + } + + off := r.BuildHuskPodForTest(pool, tmpl, controller.HuskPodOptions{StubImage: "img", MultiVM: true, PrepareEgressLink: true}) + if argsContain(off.Spec.Containers[0].Args, "--prepare-restore") { + t.Errorf("husk args must omit --prepare-restore by default: %v", off.Spec.Containers[0].Args) + } +} diff --git a/internal/controller/sandboxclaim_controller.go b/internal/controller/sandboxclaim_controller.go index 9202470c..08c279ed 100644 --- a/internal/controller/sandboxclaim_controller.go +++ b/internal/controller/sandboxclaim_controller.go @@ -159,6 +159,9 @@ type SandboxReconciler struct { // pod brings its default VM's tap up before any claim arrives, leaving only the // atomic nft policy transaction on the warm-claim hot path. Requires MultiVMFork. PrepareEgressLink bool + // PrepareRestore passes --prepare-restore so a dormant pod also loads its snapshot + // and resumes its guest before any claim arrives. Requires PrepareEgressLink. + PrepareRestore bool // spawnVM is the controller->husk spawn-vm seam used by the MultiVMFork routing. // Nil defaults to SpawnVMOnHusk; tests inject a fake. diff --git a/internal/controller/sandboxpool_controller.go b/internal/controller/sandboxpool_controller.go index 960805f8..8c803b8f 100644 --- a/internal/controller/sandboxpool_controller.go +++ b/internal/controller/sandboxpool_controller.go @@ -62,6 +62,9 @@ type SandboxPoolReconciler struct { // pod brings its default VM's tap up before any claim arrives, leaving only the // atomic nft policy transaction on the warm-claim hot path. Requires MultiVMFork. PrepareEgressLink bool + // PrepareRestore passes --prepare-restore so a dormant pod also loads its snapshot + // and resumes its guest before any claim arrives. Requires PrepareEgressLink. + PrepareRestore bool // MultiVMForkVMs is how many co-located fork VMs a multi-VM warm pod reserves // node memory for up front (beyond the source VM), so the co-location routing // has room before a fork spills to a new pod. Only consulted when MultiVM is set; diff --git a/internal/husk/multivm.go b/internal/husk/multivm.go index d0e46941..9e1faa79 100644 --- a/internal/husk/multivm.go +++ b/internal/husk/multivm.go @@ -17,6 +17,7 @@ import ( "mitos.run/mitos/internal/cas" "mitos.run/mitos/internal/firecracker" "mitos.run/mitos/internal/fork" + "mitos.run/mitos/internal/guestgrpc" "mitos.run/mitos/internal/metering" "mitos.run/mitos/internal/netconf" "mitos.run/mitos/internal/vsock" @@ -388,10 +389,80 @@ func (s *Stub) prepareInstanceOpt(ctx context.Context, id vmID, opts prepareOpts return err } + // Load the snapshot and RESUME the guest while dormant, so the claim pays only the + // fork-correctness handshake. Requires the tap prepared just above (Firecracker + // binds the baked NIC to it at restore). FAIL CLOSED: on any error we refuse to go + // dormant, so the pool never offers a half-restored pod. + if err := s.prepareRestoreDefaultVM(ctx, id, inst, opts); err != nil { + if inst.vm != nil { + _ = inst.vm.Close() + inst.vm = nil + } + return err + } + inst.state = StateDormant return nil } +// prepareRestoreDefaultVM loads the pod's default-VM snapshot and resumes its guest +// while the pod is DORMANT, when the stub opted in (Options.PrepareRestore). It is the +// same load/rootfs-rebind/resume/guest-ready the warm-claim activate does, moved off the +// hot path; activate then skips it and pays only the handshake. +// +// A NO-OP unless every precondition holds: PrepareRestore on, the tap was prepared just +// above (preparedLinkTap set), the DEFAULT vm only, a real snapshot dir configured, and +// this is not the pre-warm generic-child boot (opts.preWarmBoot). A co-located fork +// child (its own snapshot, childUFFD backend, per-fork tap) is never pre-restored here; +// it restores at spawn/activate exactly as before. +// +// The guest agent conn proven by the readiness wait is CLOSED, not held across dormancy: +// a long-idle vsock conn can go stale, and re-dialing an already-running guest at the +// claim is a sub-millisecond round trip. +func (s *Stub) prepareRestoreDefaultVM(ctx context.Context, id vmID, inst *vmInstance, opts prepareOpts) error { + // id != defaultVMID already excludes the pre-warm generic child (it is always a + // secondary VM); skipRootfsClone and reuseVM are belt-and-braces so a future + // pre-warm of the default slot never double-resumes an adopted VMM. + if !s.prepareRestore || id != defaultVMID || inst.preparedLinkTap == "" || + s.prepareSnapshotDir == "" || opts.skipRootfsClone || opts.reuseVM != nil { + return nil + } + // PrepareRestore requires PrepareEgressLink; the stub, pod builder, and controller + // all enforce that, and preparedLinkTap being set is the runtime proof the tap is up. + memFile := filepath.Join(s.prepareSnapshotDir, "mem") + vmStateFile := filepath.Join(s.prepareSnapshotDir, "vmstate") + // The baked NIC is remapped onto the prepared tap, the same override activate builds + // for the default VM (whose guest IP is the fixed in-pod constant). + overrides := []firecracker.NetworkOverride{{ + IfaceID: firecracker.NetIfaceID, + HostDevName: inst.preparedLinkTap, + }} + if err := s.setLiveCowMemSource(memFile); err != nil { + return fmt.Errorf("husk: prepare-restore arm lazy live-cow mem source for vm %q: %w", id, err) + } + if err := inst.vm.LoadSnapshotWithOverrides(memFile, vmStateFile, false, overrides); err != nil { + return fmt.Errorf("husk: prepare-restore load snapshot for vm %q: %w", id, err) + } + if inst.rootfsClonePath != "" { + if err := inst.vm.PatchDrive("rootfs", inst.rootfsClonePath); err != nil { + return fmt.Errorf("husk: prepare-restore rebind rootfs drive for vm %q: %w", id, err) + } + } + if err := inst.vm.Resume(); err != nil { + return fmt.Errorf("husk: prepare-restore resume vm %q: %w", id, err) + } + conn, err := s.ready(ctx, inst.vm.VsockHostPath(firecracker.VsockRelPath), s.readyTimeout) + if err != nil { + return fmt.Errorf("husk: prepare-restore guest not ready for vm %q: %w", id, err) + } + if conn != nil { + _ = conn.Close() //nolint:errcheck // do not hold a vsock conn across dormancy + } + inst.preRestored = true + inst.preRestoredSnapshotDir = s.prepareSnapshotDir + return nil +} + // prepareEgressLinkFor brings up the default VM's tap with a default-deny policy while // the pod is dormant, when the stub opted in. A no-op otherwise, and never for a // secondary (co-located fork child) VM, whose tap is derived per fork at activate. @@ -586,6 +657,35 @@ func (s *Stub) activateInstance(ctx context.Context, id vmID, req ActivateReques recordStage(stages, "egress_filter", mark) mark = time.Now() + // PRE-RESTORED fast path: this instance loaded its snapshot and resumed its guest + // while dormant (Options.PrepareRestore), so skip the restore, the rootfs rebind, + // the resume, and the guest-ready wait, and pay only the handshake below. FAIL + // CLOSED on a snapshot mismatch: a resumed guest cannot be reloaded, so if the claim + // names a different snapshot than the one restored at prepare, refuse rather than + // serve the wrong image. Only the default VM is ever pre-restored (see + // prepareRestoreDefaultVM), and a co-located fork child is never pre-restored. + if inst.preRestored { + if req.SnapshotDir != inst.preRestoredSnapshotDir { + werr := fmt.Errorf("husk: activate vm %q wants snapshot %q but this pod pre-restored %q; refusing to serve the wrong image", id, req.SnapshotDir, inst.preRestoredSnapshotDir) + return ActivateResult{OK: false, Error: werr.Error()}, werr + } + recordStage(stages, "vmstate_restore", mark) + recordStage(stages, "resume", mark) + // Re-dial the already-running guest for the handshake. Sub-millisecond: the + // guest agent's vsock listener has been up since the prepare-time resume. + vsockPath := inst.vm.VsockHostPath(firecracker.VsockRelPath) + guestConn, err := s.ready(ctx, vsockPath, s.readyTimeout) + if err != nil { + werr := fmt.Errorf("husk: pre-restored guest not reachable at activate for vm %q: %w", id, err) + return ActivateResult{OK: false, Error: werr.Error()}, werr + } + if guestConn != nil { + defer guestConn.Close() //nolint:errcheck // best-effort on close + } + recordStage(stages, "guest_ready", mark) + return s.activateHandshakeAndServe(ctx, id, inst, req, perNet, vsockPath, guestConn, stages, start) + } + // LAZY live-cow restore (this is what makes vmstate_restore O(1) instead of an // eager 512 MiB memfd copy inside PUT /snapshot/load): hand the armed WP handler // the mem file BEFORE the load, so it can serve the MISSING faults the patched @@ -649,8 +749,19 @@ func (s *Stub) activateInstance(ctx context.Context, id vmID, req ActivateReques defer guestConn.Close() //nolint:errcheck // best-effort on close } recordStage(stages, "guest_ready", mark) - mark = time.Now() + return s.activateHandshakeAndServe(ctx, id, inst, req, perNet, vsockPath, guestConn, stages, start) +} + +// activateHandshakeAndServe runs the CLAIM-specific tail of an activation: the fork- +// correctness handshake (fresh entropy, clock step, per-VM network re-addressing, and +// the tenant secrets), serving the sandbox API with the claim token, and the state and +// timing bookkeeping. It is shared by the normal restore-at-activate path and the +// pre-restored fast path, which reach it with an already-ready guest. mark is the clock +// for the handshake stage; start is the whole-activate clock for the total. +func (s *Stub) activateHandshakeAndServe(ctx context.Context, id vmID, inst *vmInstance, req ActivateRequest, perNet *vsock.NotifyForkedNetwork, vsockPath string, guestConn *guestgrpc.Client, stages map[string]float64, start time.Time) (ActivateResult, error) { + _ = ctx + mark := time.Now() // Fork-correctness handshake with a fresh per-VM generation + entropy. Each // instance owns its own generation counter, so two VMs never share it. The // entropy and secret values are held only in memory and are NEVER logged. diff --git a/internal/husk/multivm_network_test.go b/internal/husk/multivm_network_test.go index d598b7f8..e334c6bf 100644 --- a/internal/husk/multivm_network_test.go +++ b/internal/husk/multivm_network_test.go @@ -346,3 +346,122 @@ func TestAFailedClaimPolicyRebuildsTheLinkOnRetry(t *testing.T) { t.Error("a failed claim left activeTap set for a tap that was torn down") } } + +// --- prepare-time restore ----------------------------------------------------------- + +// newPrepareRestoreStub builds a multi-VM stub that brings its tap up AND restores + +// resumes its default VM while dormant, over a recording net runner. It configures the +// prepare snapshot dir so prepareRestoreDefaultVM engages. +func newPrepareRestoreStub(t *testing.T, rr *recordingRunner, vms map[string]*fakeVMM) *Stub { + t.Helper() + start := func(cfg firecracker.VMConfig) (vmm, error) { + vm := &fakeVMM{} + if vms != nil { + vms[cfg.ID] = vm + } + return vm, nil + } + s := New(firecracker.VMConfig{ID: "husk-test"}, Options{ + Start: start, + Ready: readyOK, + Notify: (&fakeNotifier{}).notify, + Verify: verifyOK, + MultiVM: true, + PrepareEgressLink: true, + PrepareRestore: true, + InPodGuestIP: "10.200.0.2", + InPodGatewayIP: "10.200.0.1", + PrepareSnapshotDir: "/snap", + }) + s.SetNetRunner(rr.run) + return s +} + +// TestPrepareRestoresAndResumesTheGuestWhileDormant is the core of slice 2: with +// PrepareRestore, the LoadSnapshot and Resume happen at Prepare, so a claim pays neither. +func TestPrepareRestoresAndResumesTheGuestWhileDormant(t *testing.T) { + var rr recordingRunner + vms := map[string]*fakeVMM{} + s := newPrepareRestoreStub(t, &rr, vms) + ctx := context.Background() + + if err := s.Prepare(ctx); err != nil { + t.Fatalf("Prepare: %v", err) + } + vm := vms["husk-test"] + if vm == nil { + t.Fatal("no default VM created") + } + if vm.loadCalls != 1 { + t.Errorf("Prepare loaded the snapshot %d times, want exactly 1 (restore at prepare)", vm.loadCalls) + } + if vm.resumeCalls != 1 { + t.Errorf("Prepare resumed %d times, want exactly 1 (resume at prepare)", vm.resumeCalls) + } + + // The claim must NOT load or resume again: the guest is already running. + if _, err := s.Activate(ctx, ActivateRequest{SnapshotDir: "/snap", Network: baseNet()}); err != nil { + t.Fatalf("Activate: %v", err) + } + if vm.loadCalls != 1 { + t.Errorf("Activate re-loaded the snapshot (loadCalls=%d); it must reuse the pre-restored guest", vm.loadCalls) + } + if vm.resumeCalls != 1 { + t.Errorf("Activate re-resumed the guest (resumeCalls=%d); it must reuse the pre-restored guest", vm.resumeCalls) + } +} + +// TestPrepareRestoreIsOptIn: without PrepareRestore the restore stays on the activate +// path byte-for-byte (load + resume happen at Activate, not Prepare). +func TestPrepareRestoreIsOptIn(t *testing.T) { + var rr recordingRunner + vms := map[string]*fakeVMM{} + // PrepareEgressLink on but PrepareRestore OFF. + start := func(cfg firecracker.VMConfig) (vmm, error) { + vm := &fakeVMM{} + vms[cfg.ID] = vm + return vm, nil + } + s := New(firecracker.VMConfig{ID: "husk-test"}, Options{ + Start: start, Ready: readyOK, Notify: (&fakeNotifier{}).notify, Verify: verifyOK, + MultiVM: true, PrepareEgressLink: true, InPodGuestIP: "10.200.0.2", InPodGatewayIP: "10.200.0.1", + PrepareSnapshotDir: "/snap", + }) + s.SetNetRunner(rr.run) + ctx := context.Background() + + if err := s.Prepare(ctx); err != nil { + t.Fatalf("Prepare: %v", err) + } + vm := vms["husk-test"] + if vm.loadCalls != 0 || vm.resumeCalls != 0 { + t.Fatalf("Prepare restored without opting in: load=%d resume=%d", vm.loadCalls, vm.resumeCalls) + } + if _, err := s.Activate(ctx, ActivateRequest{SnapshotDir: "/snap", Network: baseNet()}); err != nil { + t.Fatalf("Activate: %v", err) + } + if vm.loadCalls != 1 || vm.resumeCalls != 1 { + t.Errorf("Activate must do the restore when not pre-restored: load=%d resume=%d", vm.loadCalls, vm.resumeCalls) + } +} + +// TestActivateFailsClosedOnASnapshotMismatchAfterPrepareRestore: a resumed guest cannot +// be reloaded, so a claim naming a DIFFERENT snapshot than the one pre-restored must be +// refused rather than served against the wrong image. +func TestActivateFailsClosedOnASnapshotMismatchAfterPrepareRestore(t *testing.T) { + var rr recordingRunner + vms := map[string]*fakeVMM{} + s := newPrepareRestoreStub(t, &rr, vms) + ctx := context.Background() + if err := s.Prepare(ctx); err != nil { + t.Fatalf("Prepare: %v", err) + } + + res, err := s.Activate(ctx, ActivateRequest{SnapshotDir: "/a-different-snapshot", Network: baseNet()}) + if err == nil || res.OK { + t.Fatalf("Activate accepted a snapshot that differs from the pre-restored one: res=%+v err=%v", res, err) + } + if !strings.Contains(res.Error, "wrong image") { + t.Errorf("mismatch error should name the hazard, got %q", res.Error) + } +} diff --git a/internal/husk/stub.go b/internal/husk/stub.go index 5589e63f..5581d452 100644 --- a/internal/husk/stub.go +++ b/internal/husk/stub.go @@ -104,7 +104,14 @@ type vmInstance struct { // activate installs the tenant policy on it and skips the link setup. Cleared // whenever the tap is torn down, so a retry re-ensures it. preparedLinkTap string - dnsProxy *dnsproxy.Server + // preRestored is set when this instance loaded its snapshot and RESUMED its guest + // while dormant (Options.PrepareRestore), so activate skips the restore/resume/ + // guest-ready and pays only the fork-correctness handshake. preRestoredSnapshotDir + // is the snapshot the dormant restore used; activate FAILS CLOSED if the claim's + // snapshot dir differs, because a resumed guest cannot be reloaded. + preRestored bool + preRestoredSnapshotDir string + dnsProxy *dnsproxy.Server // childUFFDPlan carries a co-located live-cow fork child's LAZY UFFD import // intent from SpawnVM to activateInstance: the ChildMemfdImport coordinates (the @@ -627,6 +634,15 @@ type Options struct { // derives from the guest IP. Config, never secrets. InPodGuestIP string InPodGatewayIP string + // PrepareRestore opts a multi-VM pod's DEFAULT VM into loading its snapshot and + // resuming its guest while the pod is DORMANT, so a claim pays only the fork- + // correctness handshake instead of the snapshot restore, the resume, and the guest- + // ready wait (measured ~55 ms of the ~113 ms warm-claim activate, plus the demand + // fault-in that makes the first run_code cold). REQUIRES PrepareEgressLink (the tap + // must exist before LoadSnapshot) and InPodGuestIP. Default off. The dormant guest + // serves no tenant and is reseeded fail-closed at the claim's NotifyForked, exactly + // as a restore-at-activate guest is. See docs/superpowers/plans/2026-07-10-prepare-time-restore.md. + PrepareRestore bool // PrewarmChild opts into keeping ONE dormant, generic co-located child // Firecracker pre-prepared (booted, and snapshot-verified when a template // snapshot is configured) in a multi-VM pod, so a co-located fork that does @@ -753,6 +769,7 @@ type Stub struct { // prepareEgressLink / inPodGuestIP / inPodGatewayIP back Options.PrepareEgressLink. prepareEgressLink bool + prepareRestore bool inPodGuestIP string inPodGatewayIP string // prewarmChild keeps ONE dormant, generic co-located child Firecracker @@ -833,6 +850,7 @@ func New(cfg firecracker.VMConfig, opts Options) *Stub { liveCowFork: opts.LiveCowFork, liveCowChildImport: opts.LiveCowChildImport, prepareEgressLink: opts.PrepareEgressLink, + prepareRestore: opts.PrepareRestore, inPodGuestIP: opts.InPodGuestIP, inPodGatewayIP: opts.InPodGatewayIP, prewarmChild: opts.PrewarmChild, From 068eeca9b32c360a215e23d31827eb991515c453 Mon Sep 17 00:00:00 2001 From: Jannes Stubbemann Date: Fri, 10 Jul 2026 23:58:57 +0200 Subject: [PATCH 2/2] fix(husk): refuse a mismatched pre-restored snapshot before any side effect Review found that the snapshot-identity gate fired too late. It sat at the pre-restored fast path, AFTER the verify-on-activate re-hash and AFTER the per-VM egress filter, so a claim naming a different snapshot than the one this pod pre-restored had already had the tenant's real egress policy installed on the tap fronting the wrong guest (and a re-verify run against the mismatched dir) before the "refusing to serve the wrong image" rejection fired. The VM never went active and never got secrets or a reseed, but the network mutation happened before the gate, which is the opposite of fail-closed. The identity check now runs FIRST, right after the empty-snapshot-dir guard, before any verify or egress mutation. The fast path keeps only a comment noting the invariant now holds. TestActivateFailsClosedOnASnapshotMismatchAfterPrepareRestore now records the net command count and asserts a mismatch adds NONE; with the gate moved back to its old position the test fails with "mutated the egress datapath (1 new net commands)". Also from review: two docstrings named a nonexistent opts.preWarmBoot; the actual guard is opts.skipRootfsClone || opts.reuseVM != nil, and the docstrings now say so. Signed-off-by: Jannes Stubbemann --- internal/husk/multivm.go | 19 ++++++++++++++----- internal/husk/multivm_network_test.go | 9 +++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/internal/husk/multivm.go b/internal/husk/multivm.go index 9e1faa79..58b54beb 100644 --- a/internal/husk/multivm.go +++ b/internal/husk/multivm.go @@ -412,7 +412,8 @@ func (s *Stub) prepareInstanceOpt(ctx context.Context, id vmID, opts prepareOpts // // A NO-OP unless every precondition holds: PrepareRestore on, the tap was prepared just // above (preparedLinkTap set), the DEFAULT vm only, a real snapshot dir configured, and -// this is not the pre-warm generic-child boot (opts.preWarmBoot). A co-located fork +// this is not the pre-warm generic-child boot (opts.skipRootfsClone) or an adopted VMM +// (opts.reuseVM). A co-located fork // child (its own snapshot, childUFFD backend, per-fork tap) is never pre-restored here; // it restores at spawn/activate exactly as before. // @@ -574,6 +575,16 @@ func (s *Stub) activateInstance(ctx context.Context, id vmID, req ActivateReques return ActivateResult{OK: false, Error: "activate: empty snapshot dir"}, fmt.Errorf("husk: activate vm %q: empty snapshot dir", id) } + // Pre-restored snapshot IDENTITY gate, FIRST, before any side effect. A resumed + // guest cannot be reloaded, so a claim naming a different snapshot than the one this + // pod pre-restored must be refused BEFORE the verify-on-activate re-hash runs and + // BEFORE the tenant egress policy is installed on the tap fronting the (wrong) guest. + // Hoisting it here (rather than at the pre-restored fast path below) is what makes + // the refusal genuinely fail-closed: no network mutation, no re-verify, nothing. + if inst.preRestored && req.SnapshotDir != inst.preRestoredSnapshotDir { + werr := fmt.Errorf("husk: activate vm %q wants snapshot %q but this pod pre-restored %q; refusing to serve the wrong image", id, req.SnapshotDir, inst.preRestoredSnapshotDir) + return ActivateResult{OK: false, Error: werr.Error()}, werr + } memFile := filepath.Join(req.SnapshotDir, "mem") vmStateFile := filepath.Join(req.SnapshotDir, "vmstate") @@ -665,10 +676,8 @@ func (s *Stub) activateInstance(ctx context.Context, id vmID, req ActivateReques // serve the wrong image. Only the default VM is ever pre-restored (see // prepareRestoreDefaultVM), and a co-located fork child is never pre-restored. if inst.preRestored { - if req.SnapshotDir != inst.preRestoredSnapshotDir { - werr := fmt.Errorf("husk: activate vm %q wants snapshot %q but this pod pre-restored %q; refusing to serve the wrong image", id, req.SnapshotDir, inst.preRestoredSnapshotDir) - return ActivateResult{OK: false, Error: werr.Error()}, werr - } + // The snapshot-identity gate ran fail-closed at the top of activate, before the + // egress policy, so by here req.SnapshotDir == inst.preRestoredSnapshotDir. recordStage(stages, "vmstate_restore", mark) recordStage(stages, "resume", mark) // Re-dial the already-running guest for the handshake. Sub-millisecond: the diff --git a/internal/husk/multivm_network_test.go b/internal/husk/multivm_network_test.go index e334c6bf..1d5c65c5 100644 --- a/internal/husk/multivm_network_test.go +++ b/internal/husk/multivm_network_test.go @@ -457,6 +457,11 @@ func TestActivateFailsClosedOnASnapshotMismatchAfterPrepareRestore(t *testing.T) t.Fatalf("Prepare: %v", err) } + // Record how many net commands Prepare ran, so we can assert the mismatch refusal + // touched NONE beyond them: the identity gate fires before the egress policy, so a + // wrong-image claim must not install a tenant policy on the tap fronting the guest. + nBeforeActivate := len(rr.calls) + res, err := s.Activate(ctx, ActivateRequest{SnapshotDir: "/a-different-snapshot", Network: baseNet()}) if err == nil || res.OK { t.Fatalf("Activate accepted a snapshot that differs from the pre-restored one: res=%+v err=%v", res, err) @@ -464,4 +469,8 @@ func TestActivateFailsClosedOnASnapshotMismatchAfterPrepareRestore(t *testing.T) if !strings.Contains(res.Error, "wrong image") { t.Errorf("mismatch error should name the hazard, got %q", res.Error) } + if len(rr.calls) != nBeforeActivate { + t.Errorf("a wrong-image claim mutated the egress datapath (%d new net commands); the identity gate must refuse before any side effect: %v", + len(rr.calls)-nBeforeActivate, rr.calls[nBeforeActivate:]) + } }