refactor(husk): bring the tap up while the pod is dormant, not while a tenant waits

cmd/controller/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"flag"
+	"fmt"
 	"os"
 	"time"
 
@@ -38,6 +39,21 @@ func init() {
 	utilruntime.Must(v1.AddToScheme(scheme))
 }
 
+// validatePrepareFlags rejects prepare-time flag combinations that would silently do
+// nothing, so an operator asking for a latency optimization is never left with a pod
+// shape that cannot deliver it. It runs AFTER resolveRunMode, so enableHuskPods already
+// reflects the resolved path (false on --enable-raw-forkd or --mock).
+//
+// --husk-prepare-egress-link needs BOTH --multi-vm-fork (the stub prepares its tap only
+// on the multi-VM Prepare path) AND the husk-pods run path (the raw-forkd path never
+// creates a husk pod, so there is no pod to prepare a tap in).
+func validatePrepareFlags(prepareEgressLink, multiVMFork, enableHuskPods bool) error {
+	if prepareEgressLink && (!multiVMFork || !enableHuskPods) {
+		return fmt.Errorf("--husk-prepare-egress-link requires --multi-vm-fork AND the husk-pods run path (not --enable-raw-forkd or --mock): the tap is prepared only by a multi-VM husk pod, which the raw-forkd path never creates")
+	}
+	return nil
+}
+
 // resolveRunMode picks the single active controller run path from the flags.
 // husk pods is the pod-native default; --enable-raw-forkd selects the
 // fork-per-claim fallback, and --mock forces it too (the dev mock overlay has no
@@ -64,6 +80,7 @@ func main() {
 	var huskConnReuse bool
 	var liveCowChildImport bool
 	var prewarmChild bool
+	var prepareEgressLink bool
 	var huskStubImage string
 	var huskDNSUpstream string
 	var huskControlPort int
@@ -96,6 +113,7 @@ func main() {
 	flag.BoolVar(&multiVMFork, "multi-vm-fork", false, "EXPERIMENTAL, DEFAULT OFF: route a hosted fork child into an ADDITIONAL VM spawned INSIDE the source husk pod (the spawn-vm control op) instead of a brand-new child pod, when the source pod is multi-VM capable (its stub runs --multi-vm). OFF is byte-for-byte the current new-pod-per-fork path, so nothing changes unless you opt in AND the source pod is multi-VM capable; a non-capable source silently falls back to a new pod. This wires the routing + status (child status.pod = source pod, status.vmId = the spawned VM); the per-pod and node memory accounting that pends or spills a fork when a pod is full is a follow-up, so co-location is capped conservatively and the remainder spills to new pods. Only used with --enable-husk-pods.")
 	flag.BoolVar(&liveCowFork, "live-cow-fork", false, "EXPERIMENTAL, DEFAULT OFF: start warm husk pods with --live-cow-fork so a CO-LOCATED fork child shares the PARENT's resident guest memory (patched Firecracker memfd + userfaultfd write-protect) instead of restoring from the disk fork snapshot (milestone m4b), driving the hosted fork toward sub-100ms. SEPARATE from --multi-vm-fork so it can be deployed off and canaried independently. OFF is byte-for-byte the current disk co-location. The co-located child still falls back to the disk restore where the child-side memfd import is not yet complete, so turning it on never breaks a fork; it is a no-op off Linux (userfaultfd write-protect is Linux-only). Only meaningful with --enable-husk-pods.")
 	flag.BoolVar(&liveCowChildImport, "live-cow-child-import", false, "EXPERIMENTAL, DEFAULT OFF: with --live-cow-fork on, opt warm husk pods onto the VMSTATE-ONLY fork capture (skip the ~364ms disk mem write) so a co-located child boots its guest RAM from the source shared memfd instead of the disk fork snapshot. REQUIRES a child-side-import patched Firecracker; off keeps the armed source writing the disk mem so every child restores from disk and no fork hangs. Only meaningful with --live-cow-fork and --enable-husk-pods.")
+	flag.BoolVar(&prepareEgressLink, "husk-prepare-egress-link", false, "EXPERIMENTAL, DEFAULT OFF: warm husk pods bring their default VM's tap up while DORMANT, under a default-deny policy, so a claim pays only the atomic nft transaction that installs the tenant's policy instead of also paying the tap create. Requires --multi-vm-fork and --enable-husk-pods. Off keeps the whole filter on the activate path byte-for-byte.")
 	flag.BoolVar(&prewarmChild, "prewarm-child", false, "EXPERIMENTAL, DEFAULT OFF: keep one dormant generic co-located child Firecracker pre-prepared per multi-vm husk pod so a fork adopts the ready child (fc_boot=0, prepare off the hot path) instead of booting one at fork time. Requires --multi-vm-fork and --enable-husk-pods; a fresh slot re-warms async, a miss falls back to on-demand prepare byte-for-byte.")
 	flag.BoolVar(&huskConnReuse, "husk-conn-reuse", false, "EXPERIMENTAL, DEFAULT OFF: reuse ONE authenticated mTLS husk control connection per husk pod across control-plane RPCs (activate, fork-snapshot, spawn-vm, remove-fork-snapshot) instead of opening a fresh TCP+TLS handshake per RPC. A co-located fork does fork-snapshot then spawn-vm to the SAME source pod, so reuse saves the second full handshake and cuts the per-RPC connection-setup overhead toward zero, driving the hosted fork toward sub-100ms. OFF is byte-for-byte the current one-shot dial-per-RPC path. The husk server always supports both (a one-shot client that closes after one request and a reused connection that sends several), so this flag only changes the CONTROLLER side and can be canaried + rolled back independently. mTLS identity is verified on every dial and one request is in flight per connection, so a reused connection is neither less authenticated nor frame-interleaved (see docs/threat-model.md). Only meaningful with --enable-husk-pods.")
 	flag.StringVar(&huskStubImage, "husk-stub-image", "mitos-husk-stub:latest", "Container image that runs the dormant-VMM stub in a husk pod. Only used with --enable-husk-pods.")
@@ -138,6 +156,15 @@ func main() {
 	// (it has no KVM), so mock implies the raw-forkd path the dev overlay uses.
 	// forkd-the-builder runs regardless in both modes (it builds the snapshots).
 	enableHuskPods, rawForkd := resolveRunMode(enableHuskPods, enableRawForkd, mockMode)
+	// FAIL FAST on a flag combination that would silently do nothing. The husk stub only
+	// brings its tap up on the multi-VM Prepare path, so --husk-prepare-egress-link
+	// without --multi-vm-fork would hand the pool builder MultiVM=false while the claim
+	// reconciler believed the link was prepared. An operator who asked for a latency
+	// optimization must not be left with a pod shape that cannot deliver it.
+	if err := validatePrepareFlags(prepareEgressLink, multiVMFork, enableHuskPods); err != nil {
+		logger.Error(err, "invalid prepare-time flags")
+		os.Exit(1)
+	}
 	if rawForkd {
 		logger.Info("run path: raw-forkd (fork per claim); husk pods disabled", "reason-mock", mockMode, "reason-flag", enableRawForkd)
 	} else {
@@ -235,6 +262,7 @@ func main() {
 		LiveCowFork:               liveCowFork,
 		LiveCowChildImport:        liveCowChildImport,
 		PrewarmChild:              prewarmChild,
+		PrepareEgressLink:         prepareEgressLink,
 		HuskStubImage:             huskStubImage,
 		HuskDNSUpstream:           huskDNSUpstream,
 		KVMResourceName:           "mitos.run/kvm",
@@ -280,6 +308,7 @@ func main() {
 		HuskConns:          huskConnPool,
 		LiveCowChildImport: liveCowChildImport,
 		PrewarmChild:       prewarmChild,
+		PrepareEgressLink:  prepareEgressLink,
 		HuskControlPort:    huskControlPort,
 		HuskStubImage:      huskStubImage,
 		HuskDNSUpstream:    huskDNSUpstream,

cmd/controller/main_test.go

@@ -33,3 +33,24 @@ func TestResolveRunMode(t *testing.T) {
 		})
 	}
 }
+
+func TestValidatePrepareFlags(t *testing.T) {
+	cases := []struct {
+		name                               string
+		egress, multiVM, huskPods, wantErr bool
+	}{
+		{"off is fine", false, false, false, false},
+		{"egress with multivm and husk pods", true, true, true, false},
+		{"egress without multivm", true, false, true, true},
+		{"egress without husk pods (raw-forkd path)", true, true, false, true},
+		{"egress on raw-forkd and single-vm", true, false, false, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := validatePrepareFlags(tc.egress, tc.multiVM, tc.huskPods)
+			if (err != nil) != tc.wantErr {
+				t.Errorf("validatePrepareFlags(%v,%v,%v) err=%v, wantErr=%v", tc.egress, tc.multiVM, tc.huskPods, err, tc.wantErr)
+			}
+		})
+	}
+}

cmd/husk-stub/main.go

@@ -193,6 +193,9 @@ func run() error {
 		enableEgress       = flag.Bool("enable-egress-filter", true, "Program the in-pod nftables egress filter (default-deny + metadata block) for the activated VM. Requires NET_ADMIN in the pod netns. Default true (the husk isolation guarantee).")
 		multiVM            = flag.Bool("multi-vm", false, "EXPERIMENTAL, default false: opt into the multi-VM-per-pod execution mode (#764), running many same-tenant CoW forks inside ONE husk pod instead of one pod per VM. Increment 1 wires only a default-off scaffold; false (the default, and the only value the controller emits) keeps the single-VM behavior byte-for-byte. Not a secret.")
 		liveCowFork        = flag.Bool("live-cow-fork", false, "EXPERIMENTAL, default false: opt a CO-LOCATED fork child onto the live copy-on-write path (share the PARENT's resident guest memory via the patched Firecracker memfd + userfaultfd write-protect handler) instead of restoring from the disk fork snapshot (milestone m4b). SEPARATE from --multi-vm so it can be deployed off and canaried independently. Off (the default, and the only value the controller emits unless the operator opts in) keeps the co-located fork on the disk snapshot restore byte-for-byte. No-op off Linux (userfaultfd write-protect is Linux-only), failing closed to the disk restore. Not a secret.")
+		prepareEgressLink  = flag.Bool("prepare-egress-link", false, "EXPERIMENTAL, DEFAULT OFF: bring the pod's default-VM tap up while the pod is DORMANT, under a default-deny policy, so a claim pays only the atomic nft transaction that installs the tenant's policy instead of also paying the tap create (measured: the link half is about two thirds of the egress_filter stage). Requires --multi-vm and --in-pod-guest-ip. Off keeps the whole filter on the activate path byte-for-byte. Not a secret.")
+		inPodGuestIP       = flag.String("in-pod-guest-ip", "", "the fixed in-pod guest IP of this pod's default VM (the same value the controller sends in the activate request). Needed BEFORE that request arrives by --prepare-egress-link, because the tap name derives from it. Config, not a secret.")
+		inPodGatewayIP     = flag.String("in-pod-gateway-ip", "", "the fixed in-pod gateway IP paired with --in-pod-guest-ip. Config, not a secret.")
 		prewarmChild       = flag.Bool("prewarm-child", false, "EXPERIMENTAL, DEFAULT OFF: keep ONE dormant generic co-located child Firecracker pre-prepared per multi-vm pod so a fork ADOPTS the ready child (fc_boot=0, prepare off the hot path) instead of booting one at fork time. Requires --multi-vm; a fresh slot re-warms async, a miss falls back to the on-demand prepare. Not a secret.")
 		liveCowChildImport = flag.Bool("live-cow-child-import", false, "EXPERIMENTAL, default false: opt the live-cow fork onto the VMSTATE-ONLY capture (skip the ~364ms disk mem write). REQUIRES a shipped child-side memfd-import Firecracker patch: the co-located child boots its guest RAM from the source shared memfd instead of the disk fork snapshot. The currently shipped patched binary patches the SOURCE (restore) side only, so leave this OFF; with it off an armed --live-cow-fork source still writes the disk mem, so every co-located child restores from disk and the fork never hangs. Turn on ONLY once a child-side import binary is shipped. Not a secret.")
 	)
@@ -437,6 +440,9 @@ func run() error {
 		// the fork never hangs. Turn on ONLY once a child-side import binary ships.
 		LiveCowChildImport: *liveCowChildImport,
 		PrewarmChild:       *prewarmChild,
+		PrepareEgressLink:  *prepareEgressLink,
+		InPodGuestIP:       *inPodGuestIP,
+		InPodGatewayIP:     *inPodGatewayIP,
 	})
 	// Publish the constructed stub to the already-serving metering source.
 	publishedStub.Store(stub)

docs/superpowers/plans/2026-07-10-prepare-time-restore.md

@@ -0,0 +1,139 @@
+# Prepare-time restore: move the snapshot restore off the warm-claim hot path
+
+Status: slice 1 implemented (default off, not yet canaried on prod); slices 2 and 3 designed.
+
+## Why
+
+A warm claim currently pays the whole microVM restore while a tenant waits. Measured on
+prod (`mitos-kvm-1`), one matched claim, controller stage log against the husk stub's own:
+
+| stage | ms |
+|---|---|
+| controller `activate_rpc` | 129.7 |
+| husk-reported activate total | 112.8 |
+| ... `guest_ready` | 40.6 |
+| ... `egress_filter` | 29.6 |
+| ... `vmstate_restore` | 22.9 |
+| ... `handshake` | 17.2 |
+| ... `resume` | 2.4 |
+| RPC + mTLS overhead between the two | ~17 |
+
+Under sustained load the same stages mean about 62 ms in total, so treat the numbers
+above as one cold-ish sample and the ordering, not the absolute values, as the signal.
+
+Everything except `handshake` is work the pod could have done while it sat dormant with
+no tenant attached. `guest_ready` is largely demand fault-in of guest RAM under the lazy
+UFFD restore (see `bench/results/2026-07-09-lazy-livecow-restore.md`), and it is also why
+the first `run_code` in a fresh sandbox costs ~105 ms against ~35 ms warm: the ipykernel
+is already running in the snapshot (`warmKernel: true`), but its pages are not resident.
+
+Time to Interactive on the hosted API is 307.7 ms P50, of which create is ~196 ms and the
+first exec ~118 ms (`bench/results/2026-07-10-tti-hosted.md`). This is the single largest
+lever on both halves.
+
+## What is actually claim-specific
+
+Reading `activateInstance` (`internal/husk/multivm.go`), only three things depend on the
+claim:
+
+1. the tenant's netfilter policy (`req.Egress`, `req.Allow`, `req.AllowCIDRs`,
+   `req.Inbound`, `req.BlockNetwork`);
+2. the fork-correctness handshake, which carries fresh entropy, the clock step, the
+   guest's network re-addressing, and the tenant's secrets (`notifyReq`);
+3. `onActivated` (serving the sandbox API with the claim's token).
+
+Everything else is a function of the pod and its pool template:
+
+- the tap name is `netconf.DeriveTapName(guestIP)`, and the in-pod guest IP is the FIXED
+  constant `10.200.0.2` (`huskGuestIP`, `internal/controller/sandboxclaim_controller.go`);
+- the snapshot dir and its expected digest are already known at Prepare
+  (`PrepareSnapshotDir` / `PrepareExpectedDigest`), which is why Prepare already
+  pre-pays the content-addressed verify;
+- the rootfs CoW clone is per-pod;
+- the live-cow write-protect arm already happens in `prepareInstance`.
+
+So the restore itself (`setLiveCowMemSource` -> `LoadSnapshot` -> `PatchDrive` ->
+`Resume` -> guest-ready) needs nothing from the claim except a tap to bind the baked NIC
+to, and the tap name is knowable at Prepare.
+
+## Target
+
+Prepare (dormant, no tenant):
+
+1. ensure the tap exists and install a DEFAULT-DENY netfilter policy on it (a dormant VM
+   must reach nothing, fail closed);
+2. `setLiveCowMemSource`, `LoadSnapshotWithOverrides` (paused), `PatchDrive` to the CoW
+   clone, `Resume`;
+3. wait for the guest agent (this is where the demand fault-in is paid);
+4. optionally prefault the run_code kernel's working set by running one inert cell, the
+   same trick the template build uses (`warmKernelCode`).
+
+Activate (claim):
+
+1. replace the netfilter policy with the tenant's, atomically, on the tap that already
+   exists (one `nft -f -`, not the tap create plus policy);
+2. the fork-correctness handshake (entropy, clock, network, secrets);
+3. `onActivated`.
+
+Expected activate: roughly `nft` + `handshake` + `serve_api`, i.e. ~30 ms against 62 to
+113 ms today, plus the first `run_code` arriving warm.
+
+## Guards, in order of how badly they bite
+
+- **Snapshot identity.** If `req.SnapshotDir` is not the dir Prepare restored, the pod
+  has already restored the wrong image. FAIL CLOSED with a named error; never silently
+  reload, because a resumed VM cannot be reloaded.
+- **Fork children.** A co-located fork child restores a node-local fork snapshot through
+  `spawnVM` -> `prepareInstance` -> `activateInstance` with `req.ForkSnapshot` set. It
+  must keep the current path exactly. Prepare-time restore applies ONLY to
+  `defaultVMID` with `PrepareSnapshotDir` set and no fork snapshot.
+- **Dormant egress.** Between the restore and the claim, a live guest runs behind a tap.
+  It must be default-deny for that whole window, and the claim must be able to WIDEN it
+  atomically without a window where no policy is installed. `nft -f -` replaces the
+  ruleset in one transaction, which is the property we need.
+- **Reseed.** The dormant guest holds the snapshot's CRNG until the claim's
+  `NotifyForked`. It serves no tenant before then, and the handshake still gates Ready
+  fail-closed. The dormancy window only makes the clock step larger, and the step is
+  absolute.
+- **Teardown.** `Close` must remove a tap created at Prepare, not only one created at
+  Activate (`inst.activeTap` is already the seam).
+- **Memory.** A restored, resumed, prefaulted dormant VM holds its working set resident:
+  about 72 MiB per sandbox measured on the lazy path, against ~0 for a dormant VMM that
+  never loaded. At `warm.min: 8` that is roughly 576 MiB per node. The husk pod memory
+  request is already sized worst-case (`guestRAM * (1 + reserved forks)`), so this fits,
+  but it must be stated and watched.
+- **Rollout.** Default OFF behind a stub flag. Canary ONE dormant pod (delete a single
+  dormant husk pod, check `restartCount=0`, claim it, compare stages) before recycling
+  the pool. `git merge-base --is-ancestor <deployed-tag> HEAD` before building anything
+  for prod.
+
+## Slices
+
+1. **Tap at Prepare.** DONE, default off. `applyEgressFilter` is split into
+   `ensureEgressLink` (forwarding, link delete, `ip -batch`) and `applyEgressPolicy` (one
+   atomic `nft -f -`), and the composed call still runs exactly the same commands in the
+   same order (pinned by `TestApplyEgressFilterIsTheTwoHalves`). Behind
+   `--husk-prepare-egress-link` the pod ensures the tap with a default-deny policy while
+   dormant, and the claim installs the tenant policy on it: exactly one command on the hot
+   path (`TestPrepareBringsTheTapUpDormantAndActivateOnlyInstallsThePolicy`). A rejected
+   claim policy tears the tap down and clears the prepared marker, so the retry rebuilds
+   the link rather than installing a policy on a tap that no longer exists
+   (`TestAFailedClaimPolicyRebuildsTheLinkOnRetry`). Worth roughly 20 ms, and a
+   prerequisite for slice 2 because Firecracker requires the tap to exist at restore time.
+   Requires `--multi-vm-fork`; the controller refuses the combination at startup and the pod
+   builder refuses it again. Still to do: measure it on prod behind a one-pod canary, then a
+   KVM gate. Until both, the threat model does NOT call this path verified.
+2. **Restore at Prepare.** Move `setLiveCowMemSource` / `LoadSnapshot` / `PatchDrive` /
+   `Resume` / guest-ready into `prepareInstance`, behind the same flag, with the guards
+   above. Activate becomes policy + handshake + serve_api.
+3. **Prefault the kernel.** Run the inert warm cell at Prepare so the first tenant
+   `run_code` does not fault the ipykernel's pages in.
+
+Each slice ships with its own KVM gate in `firecracker-test`, because none of this is
+observable without `/dev/kvm` and the mitos-patched Firecracker.
+
+## Not in scope
+
+`mark_pod_claimed` (16.5 ms) is the optimistic-lock mutual exclusion that stops two
+claims taking one VM. Taking it off the hot path needs a reservation protocol, not a
+reordering, and is tracked separately.