mitos-run · stubbi · Jul 10, 2026 · Jul 10, 2026 · Jul 10, 2026
@@ -46,11 +46,18 @@ func init() {
 //
 // --husk-prepare-egress-link needs BOTH --multi-vm-fork (the stub prepares its tap only
 // on the multi-VM Prepare path) AND the husk-pods run path (the raw-forkd path never
-// creates a husk pod, so there is no pod to prepare a tap in).
-func validatePrepareFlags(prepareEgressLink, multiVMFork, enableHuskPods bool) error {
+// creates a husk pod, so there is no pod to prepare a tap in). --husk-prepare-restore
+// needs --husk-prepare-egress-link, and therefore everything that needs.
+func validatePrepareFlags(prepareEgressLink, prepareRestore, multiVMFork, enableHuskPods bool) error {
 	if prepareEgressLink && (!multiVMFork || !enableHuskPods) {
 		return fmt.Errorf("--husk-prepare-egress-link requires --multi-vm-fork AND the husk-pods run path (not --enable-raw-forkd or --mock): the tap is prepared only by a multi-VM husk pod, which the raw-forkd path never creates")
 	}
+	// Prepare-time restore builds ON the prepared tap (Firecracker binds the baked NIC
+	// to it at snapshot load), so it requires the egress link, and transitively
+	// everything the egress link requires.
+	if prepareRestore && !prepareEgressLink {
+		return fmt.Errorf("--husk-prepare-restore requires --husk-prepare-egress-link: the tap must exist before the snapshot load")
+	}
 	return nil
 }
 
@@ -81,6 +88,7 @@ func main() {
 	var liveCowChildImport bool
 	var prewarmChild bool
 	var prepareEgressLink bool
+	var prepareRestore bool
 	var huskStubImage string
 	var huskDNSUpstream string
 	var huskControlPort int
@@ -113,6 +121,7 @@ func main() {
 	flag.BoolVar(&multiVMFork, "multi-vm-fork", false, "EXPERIMENTAL, DEFAULT OFF: route a hosted fork child into an ADDITIONAL VM spawned INSIDE the source husk pod (the spawn-vm control op) instead of a brand-new child pod, when the source pod is multi-VM capable (its stub runs --multi-vm). OFF is byte-for-byte the current new-pod-per-fork path, so nothing changes unless you opt in AND the source pod is multi-VM capable; a non-capable source silently falls back to a new pod. This wires the routing + status (child status.pod = source pod, status.vmId = the spawned VM); the per-pod and node memory accounting that pends or spills a fork when a pod is full is a follow-up, so co-location is capped conservatively and the remainder spills to new pods. Only used with --enable-husk-pods.")
 	flag.BoolVar(&liveCowFork, "live-cow-fork", false, "EXPERIMENTAL, DEFAULT OFF: start warm husk pods with --live-cow-fork so a CO-LOCATED fork child shares the PARENT's resident guest memory (patched Firecracker memfd + userfaultfd write-protect) instead of restoring from the disk fork snapshot (milestone m4b), driving the hosted fork toward sub-100ms. SEPARATE from --multi-vm-fork so it can be deployed off and canaried independently. OFF is byte-for-byte the current disk co-location. The co-located child still falls back to the disk restore where the child-side memfd import is not yet complete, so turning it on never breaks a fork; it is a no-op off Linux (userfaultfd write-protect is Linux-only). Only meaningful with --enable-husk-pods.")
 	flag.BoolVar(&liveCowChildImport, "live-cow-child-import", false, "EXPERIMENTAL, DEFAULT OFF: with --live-cow-fork on, opt warm husk pods onto the VMSTATE-ONLY fork capture (skip the ~364ms disk mem write) so a co-located child boots its guest RAM from the source shared memfd instead of the disk fork snapshot. REQUIRES a child-side-import patched Firecracker; off keeps the armed source writing the disk mem so every child restores from disk and no fork hangs. Only meaningful with --live-cow-fork and --enable-husk-pods.")
+	flag.BoolVar(&prepareRestore, "husk-prepare-restore", false, "EXPERIMENTAL, DEFAULT OFF: warm husk pods load their default VM's snapshot and RESUME its guest while DORMANT, so a claim pays only the fork-correctness handshake instead of the restore, resume, and guest-ready wait. REQUIRES --husk-prepare-egress-link (the tap must exist before the snapshot load) and --multi-vm-fork. Off keeps the whole restore on the activate path byte-for-byte.")
 	flag.BoolVar(&prepareEgressLink, "husk-prepare-egress-link", false, "EXPERIMENTAL, DEFAULT OFF: warm husk pods bring their default VM's tap up while DORMANT, under a default-deny policy, so a claim pays only the atomic nft transaction that installs the tenant's policy instead of also paying the tap create. Requires --multi-vm-fork and --enable-husk-pods. Off keeps the whole filter on the activate path byte-for-byte.")
 	flag.BoolVar(&prewarmChild, "prewarm-child", false, "EXPERIMENTAL, DEFAULT OFF: keep one dormant generic co-located child Firecracker pre-prepared per multi-vm husk pod so a fork adopts the ready child (fc_boot=0, prepare off the hot path) instead of booting one at fork time. Requires --multi-vm-fork and --enable-husk-pods; a fresh slot re-warms async, a miss falls back to on-demand prepare byte-for-byte.")
 	flag.BoolVar(&huskConnReuse, "husk-conn-reuse", false, "EXPERIMENTAL, DEFAULT OFF: reuse ONE authenticated mTLS husk control connection per husk pod across control-plane RPCs (activate, fork-snapshot, spawn-vm, remove-fork-snapshot) instead of opening a fresh TCP+TLS handshake per RPC. A co-located fork does fork-snapshot then spawn-vm to the SAME source pod, so reuse saves the second full handshake and cuts the per-RPC connection-setup overhead toward zero, driving the hosted fork toward sub-100ms. OFF is byte-for-byte the current one-shot dial-per-RPC path. The husk server always supports both (a one-shot client that closes after one request and a reused connection that sends several), so this flag only changes the CONTROLLER side and can be canaried + rolled back independently. mTLS identity is verified on every dial and one request is in flight per connection, so a reused connection is neither less authenticated nor frame-interleaved (see docs/threat-model.md). Only meaningful with --enable-husk-pods.")
@@ -161,7 +170,7 @@ func main() {
 	// without --multi-vm-fork would hand the pool builder MultiVM=false while the claim
 	// reconciler believed the link was prepared. An operator who asked for a latency
 	// optimization must not be left with a pod shape that cannot deliver it.
-	if err := validatePrepareFlags(prepareEgressLink, multiVMFork, enableHuskPods); err != nil {
+	if err := validatePrepareFlags(prepareEgressLink, prepareRestore, multiVMFork, enableHuskPods); err != nil {
 		logger.Error(err, "invalid prepare-time flags")
 		os.Exit(1)
 	}
@@ -263,6 +272,7 @@ func main() {
 		LiveCowChildImport:        liveCowChildImport,
 		PrewarmChild:              prewarmChild,
 		PrepareEgressLink:         prepareEgressLink,
+		PrepareRestore:            prepareRestore,
 		HuskStubImage:             huskStubImage,
 		HuskDNSUpstream:           huskDNSUpstream,
 		KVMResourceName:           "mitos.run/kvm",
@@ -309,6 +319,7 @@ func main() {
 		LiveCowChildImport: liveCowChildImport,
 		PrewarmChild:       prewarmChild,
 		PrepareEgressLink:  prepareEgressLink,
+		PrepareRestore:     prepareRestore,
 		HuskControlPort:    huskControlPort,
 		HuskStubImage:      huskStubImage,
 		HuskDNSUpstream:    huskDNSUpstream,

@@ -36,20 +36,22 @@ func TestResolveRunMode(t *testing.T) {
 
 func TestValidatePrepareFlags(t *testing.T) {
 	cases := []struct {
-		name                               string
-		egress, multiVM, huskPods, wantErr bool
+		name                                        string
+		egress, restore, multiVM, huskPods, wantErr bool
 	}{
-		{"off is fine", false, false, false, false},
-		{"egress with multivm and husk pods", true, true, true, false},
-		{"egress without multivm", true, false, true, true},
-		{"egress without husk pods (raw-forkd path)", true, true, false, true},
-		{"egress on raw-forkd and single-vm", true, false, false, true},
+		{"off is fine", false, false, false, false, false},
+		{"egress with multivm and husk pods", true, false, true, true, false},
+		{"egress without multivm", true, false, false, true, true},
+		{"egress without husk pods (raw-forkd path)", true, false, true, false, true},
+		{"restore with egress, multivm, husk pods", true, true, true, true, false},
+		{"restore without egress", false, true, true, true, true},
+		{"restore with egress but no multivm", true, true, false, true, true},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
-			err := validatePrepareFlags(tc.egress, tc.multiVM, tc.huskPods)
+			err := validatePrepareFlags(tc.egress, tc.restore, tc.multiVM, tc.huskPods)
 			if (err != nil) != tc.wantErr {
-				t.Errorf("validatePrepareFlags(%v,%v,%v) err=%v, wantErr=%v", tc.egress, tc.multiVM, tc.huskPods, err, tc.wantErr)
+				t.Errorf("validatePrepareFlags(egress=%v,restore=%v,multiVM=%v,huskPods=%v) err=%v, wantErr=%v", tc.egress, tc.restore, tc.multiVM, tc.huskPods, err, tc.wantErr)
 			}
 		})
 	}

@@ -196,6 +196,7 @@ func run() error {
 		prepareEgressLink  = flag.Bool("prepare-egress-link", false, "EXPERIMENTAL, DEFAULT OFF: bring the pod's default-VM tap up while the pod is DORMANT, under a default-deny policy, so a claim pays only the atomic nft transaction that installs the tenant's policy instead of also paying the tap create (measured: the link half is about two thirds of the egress_filter stage). Requires --multi-vm and --in-pod-guest-ip. Off keeps the whole filter on the activate path byte-for-byte. Not a secret.")
 		inPodGuestIP       = flag.String("in-pod-guest-ip", "", "the fixed in-pod guest IP of this pod's default VM (the same value the controller sends in the activate request). Needed BEFORE that request arrives by --prepare-egress-link, because the tap name derives from it. Config, not a secret.")
 		inPodGatewayIP     = flag.String("in-pod-gateway-ip", "", "the fixed in-pod gateway IP paired with --in-pod-guest-ip. Config, not a secret.")
+		prepareRestore     = flag.Bool("prepare-restore", false, "EXPERIMENTAL, DEFAULT OFF: load the default VM's snapshot and RESUME its guest while the pod is DORMANT, so a claim pays only the fork-correctness handshake instead of the restore, resume, and guest-ready wait. REQUIRES --prepare-egress-link (the tap must exist before the snapshot load). Off keeps the whole restore on the activate path byte-for-byte. Not a secret.")
 		prewarmChild       = flag.Bool("prewarm-child", false, "EXPERIMENTAL, DEFAULT OFF: keep ONE dormant generic co-located child Firecracker pre-prepared per multi-vm pod so a fork ADOPTS the ready child (fc_boot=0, prepare off the hot path) instead of booting one at fork time. Requires --multi-vm; a fresh slot re-warms async, a miss falls back to the on-demand prepare. Not a secret.")
 		liveCowChildImport = flag.Bool("live-cow-child-import", false, "EXPERIMENTAL, default false: opt the live-cow fork onto the VMSTATE-ONLY capture (skip the ~364ms disk mem write). REQUIRES a shipped child-side memfd-import Firecracker patch: the co-located child boots its guest RAM from the source shared memfd instead of the disk fork snapshot. The currently shipped patched binary patches the SOURCE (restore) side only, so leave this OFF; with it off an armed --live-cow-fork source still writes the disk mem, so every co-located child restores from disk and the fork never hangs. Turn on ONLY once a child-side import binary is shipped. Not a secret.")
 	)
@@ -441,6 +442,7 @@ func run() error {
 		LiveCowChildImport: *liveCowChildImport,
 		PrewarmChild:       *prewarmChild,
 		PrepareEgressLink:  *prepareEgressLink,
+		PrepareRestore:     *prepareRestore,
 		InPodGuestIP:       *inPodGuestIP,
 		InPodGatewayIP:     *inPodGatewayIP,
 	})

@@ -1,6 +1,6 @@
 # Prepare-time restore: move the snapshot restore off the warm-claim hot path
 
-Status: slice 1 implemented (default off, not yet canaried on prod); slices 2 and 3 designed.
+Status: slices 1 and 2 implemented (default off, not yet canaried on prod); slice 3 designed.
 
 ## Why
 
@@ -123,9 +123,18 @@ Expected activate: roughly `nft` + `handshake` + `serve_api`, i.e. ~30 ms agains
    Requires `--multi-vm-fork`; the controller refuses the combination at startup and the pod
    builder refuses it again. Still to do: measure it on prod behind a one-pod canary, then a
    KVM gate. Until both, the threat model does NOT call this path verified.
-2. **Restore at Prepare.** Move `setLiveCowMemSource` / `LoadSnapshot` / `PatchDrive` /
-   `Resume` / guest-ready into `prepareInstance`, behind the same flag, with the guards
-   above. Activate becomes policy + handshake + serve_api.
+2. **Restore at Prepare.** DONE, default off. Behind `--husk-prepare-restore` (which
+   requires `--husk-prepare-egress-link`), `prepareRestoreDefaultVM` runs
+   `setLiveCowMemSource` / `LoadSnapshotWithOverrides` / `PatchDrive` / `Resume` /
+   guest-ready for the DEFAULT VM after the tap is up, closing the ready conn rather than
+   holding it across dormancy. Activate takes a fast path when `inst.preRestored`: it
+   skips the restore, re-dials the already-running guest, and pays only the handshake via
+   the shared `activateHandshakeAndServe`. FAIL CLOSED on a snapshot-dir mismatch (a
+   resumed guest cannot be reloaded). Co-located fork children and the pre-warm child are
+   never pre-restored. Pinned by `TestPrepareRestoresAndResumesTheGuestWhileDormant`,
+   `TestPrepareRestoreIsOptIn`, `TestActivateFailsClosedOnASnapshotMismatchAfterPrepareRestore`,
+   `TestBuildHuskPodThreadsPrepareRestore`. Still to do: a KVM gate and a one-pod prod
+   canary before it is called verified.
 3. **Prefault the kernel.** Run the inert warm cell at Prepare so the first tenant
    `run_code` does not fault the ipykernel's pages in.