diff --git a/cmd/forkd/jailer.go b/cmd/forkd/jailer.go index 694beb0c..bbc3d575 100644 --- a/cmd/forkd/jailer.go +++ b/cmd/forkd/jailer.go @@ -87,6 +87,13 @@ func parseUIDRange(s string) (uint32, uint32, error) { if low > high { return 0, 0, fmt.Errorf("--uid-range %q: low bound above high bound", s) } + // The shared kvm gid owns the group-readable template artifacts so a husk + // reads them through the group class. A per-VM jailer uid/gid drawn from a + // range that covers it would collide with that shared group, so a jailed + // build VM could carry the template-read group. Refuse such a range. + if low <= firecracker.SharedKVMGID && firecracker.SharedKVMGID <= high { + return 0, 0, fmt.Errorf("--uid-range %q: range covers the shared kvm gid %d, reserved for template-read group ownership; choose a range that excludes it", s, firecracker.SharedKVMGID) + } return uint32(low), uint32(high), nil } diff --git a/cmd/forkd/jailer_test.go b/cmd/forkd/jailer_test.go index dc10ccc9..d4f8caf0 100644 --- a/cmd/forkd/jailer_test.go +++ b/cmd/forkd/jailer_test.go @@ -22,6 +22,8 @@ func TestParseUIDRange(t *testing.T) { {in: "-64999", wantErr: true}, {in: "64999-64000", wantErr: true}, // low above high {in: "0-100", wantErr: true}, // uid 0 is root; never jail as root + {in: "64500-65500", wantErr: true}, // covers the shared kvm gid 65000 + {in: "65000-65000", wantErr: true}, // exactly the shared kvm gid {in: "", wantErr: true}, } for _, c := range cases { diff --git a/docs/threat-model.md b/docs/threat-model.md index 0daf93c4..c4f77943 100644 --- a/docs/threat-model.md +++ b/docs/threat-model.md @@ -348,6 +348,33 @@ tenant's filesystem state into another. The clone is removed on pod teardown (`Stub.Close`). Fully pod-native snapshot delivery (a CAS pull into the pod, removing the shared read-only mem/vmstate hostPath) remains a documented follow-up. +Template artifact FILE PERMISSION posture (#597). The jailed build leaves the +canonical template artifacts (`rootfs.ext4`, `snapshot/mem`, `snapshot/vmstate`) +owned by the per-VM jailer build uid (`firecracker.JailerBuildUID`, the low end +of `--uid-range`), which a husk VMM that is neither that uid nor privileged +cannot open. `internal/firecracker` `normalizeTemplateArtifacts` now hands them +back to `root` and the shared kvm group (`firecracker.SharedKVMGID`, a fixed gid +OUTSIDE the per-VM jailer range so it can never collide with a per-VM jailer +gid), with group-readable, NOT world-writable modes (files `0o640`, dirs +`0o750`); the prior posture was world-readable `0o644`/`0o755`. The husk pod +carries `SharedKVMGID` as a supplemental group (`huskpod.go` PodSecurityContext), +so the current uid-0 husk reads the artifacts as the file OWNER and a future +non-root husk (#585) reads them through the file GROUP class. `internal/fork` +`checkTemplateArtifactInvariants` is the read-side reuse gate: it refuses to +reuse an on-disk template whose ownership regressed off `root:SharedKVMGID`, +group-readable. The artifacts stay read-only to the husk (its per-activation +reflink clone is the only writable copy) and the template hostPath stays a +read-write MOUNT only because Firecracker opens the baked rootfs O_RDWR at load +(above); the FILES themselves are never world-writable. The husk now +FAILS CLOSED when a template rootfs is configured without a per-activation CoW +clone rather than restoring the shared template in place (`Stub.Activate`, +mandatory CoW), so a misconfiguration can never silently reopen the shared +`rootfs.ext4` O_RDWR. Residual, stated honestly: because the baked rootfs load is +O_RDWR, a FUTURE non-root husk (#585) reading through the group class will still +need the load itself resolved (group-write on the template, or a read-only baked +rootfs drive); this PR keeps the current uid-0 husk correct as the file owner and +lands the group-read contract the non-root husk will build on. + ### Warm-pool autoscaling (no integrity-gate move) Demand-driven warm-pool autoscaling changes only WHEN and HOW MANY dormant husk diff --git a/internal/controller/huskpod.go b/internal/controller/huskpod.go index 2252a555..6a539f1d 100644 --- a/internal/controller/huskpod.go +++ b/internal/controller/huskpod.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" v1 "mitos.run/mitos/api/v1" + "mitos.run/mitos/internal/firecracker" "mitos.run/mitos/internal/tenant" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -885,6 +886,17 @@ func (r *SandboxPoolReconciler) buildHuskPod(pool *v1.SandboxPool, template *v1. SeccompProfile: &corev1.SeccompProfile{ Type: corev1.SeccompProfileTypeRuntimeDefault, }, + // The shared kvm group the template artifacts are group-owned by + // (forkd's normalizeTemplateArtifacts writes them root:SharedKVMGID, + // group-readable). Carrying it as a supplemental group means every + // stub process is in the file group class of the template rootfs, + // snapshot mem, and vmstate, so a future non-root husk (issue #585) + // reads the shared template through the group without owning it. The + // current uid-0 husk already reads them as owner; adding the group is + // a no-op for it and does not widen anything (issue #597). Not applied + // as fsGroup: hostPath volumes are exempt from fsGroup ownership + // management, so it would neither help nor take effect there. + SupplementalGroups: []int64{firecracker.SharedKVMGID}, }, // Pin to a KVM node: the dormant VMM needs /dev/kvm AND the pod must // land where the template snapshot hostPath exists. The nodeAffinity diff --git a/internal/controller/huskpod_test.go b/internal/controller/huskpod_test.go index 725555e0..9fca874c 100644 --- a/internal/controller/huskpod_test.go +++ b/internal/controller/huskpod_test.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "mitos.run/mitos/internal/controller" + "mitos.run/mitos/internal/firecracker" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -316,6 +317,19 @@ func TestBuildHuskPodPSARestricted(t *testing.T) { if psc.RunAsNonRoot == nil || *psc.RunAsNonRoot { t.Error("pod RunAsNonRoot must be explicitly false (the documented /dev/kvm device exception)") } + // The husk pod carries the shared kvm gid as a supplemental group so a future + // non-root husk (issue #585) reads the group-owned template artifacts through + // the group class (issue #597). The current uid-0 husk reads them as owner; + // the group is additive and never widens access. + foundKVMGroup := false + for _, g := range psc.SupplementalGroups { + if g == firecracker.SharedKVMGID { + foundKVMGroup = true + } + } + if !foundKVMGroup { + t.Errorf("pod SupplementalGroups = %v, want to include the shared kvm gid %d so the husk can read the group-owned template artifacts", psc.SupplementalGroups, firecracker.SharedKVMGID) + } // CONTAINER-LEVEL securityContext: every other restricted control IS satisfied, // so the husk pod's securityContext is restricted-clean and only the hostPath + diff --git a/internal/controller/pool_rebuild.go b/internal/controller/pool_rebuild.go index 4a673a91..3e39fc55 100644 --- a/internal/controller/pool_rebuild.go +++ b/internal/controller/pool_rebuild.go @@ -140,6 +140,16 @@ func templateRestoreFailing(pods []corev1.Pod, digest string) bool { func (r *SandboxPoolReconciler) driveTemplateHealth(ctx context.Context, pool *v1.SandboxPool, template *v1.PoolTemplateSpec, templateID string, nodeFilter map[string]bool, dormantPods []corev1.Pod, warmReady bool, now metav1.Time) { logger := log.FromContext(ctx) digest := pool.Status.TemplateDigest + // TODO(#578, #579): templateRestoreFailing detects husk pods that CRASHLOOP + // (e.g. a Prepare-time reflink clone or snapshot verify that fails closed), so + // a template that is unreadable at Prepare is already reflected here. It does + // NOT yet cover a pod that stays dormant+Ready but fails every ACTIVATE (e.g. + // the mandatory-CoW fail-closed on a missing clone, or a filesystem + // permission error surfaced only at load): the pool can still report Ready + // while 100% of activations fail. Surfacing per-activation failures on the + // SandboxPool status is the #578 honesty gap and is deferred to that issue; + // #597 closes the read/CoW permission root cause so this path fails at Prepare + // (visible) rather than silently at Activate wherever possible. failing := templateRestoreFailing(dormantPods, digest) if !failing { diff --git a/internal/firecracker/jailer.go b/internal/firecracker/jailer.go index 2b37bbc1..16cb6b55 100644 --- a/internal/firecracker/jailer.go +++ b/internal/firecracker/jailer.go @@ -27,6 +27,27 @@ const jailerExecFileName = "firecracker" // binds the socket there. const jailedAPISocketRelPath = "/run/firecracker.socket" +// JailerBuildUID is the low end of the default per-VM jailer uid/gid range +// (cmd/forkd --uid-range "64000-64999"). The jailed template build VM runs as +// this uid, and the build hardlinks the canonical template artifacts into its +// chroot; a freshly built, un-normalized template therefore comes out owned by +// this uid, which a husk VMM that is neither this uid nor privileged cannot +// open (issues #583, #597). It is named here so the ownership normalize +// (normalizeTemplateArtifacts) and the read-side reuse invariant +// (internal/fork.checkTemplateArtifactInvariants) refer to the same value. +const JailerBuildUID = 64000 + +// SharedKVMGID is the group the normalized template artifacts are owned by so a +// husk VMM can READ them (rootfs.ext4 for the per-activation reflink clone, +// snapshot mem and vmstate for the load) through the file group class, without +// being the file owner. It is chosen OUTSIDE the per-VM jailer uid/gid range +// [64000, 64999] so it can never collide with a per-VM jailer gid. The husk pod +// carries it as a supplemental group (see the controller husk pod builder), so +// once the husk runs a non-root uid (issue #585) it still reads the shared +// template artifacts through this group. The current uid-0 husk owns the +// artifacts directly (they are root-owned after normalize) and is unaffected. +const SharedKVMGID = 65000 + // JailerConfig configures launching Firecracker through the jailer // binary. The zero value disables the jailer: StartVM execs the // firecracker binary directly, exactly as before. diff --git a/internal/firecracker/template.go b/internal/firecracker/template.go index 012ed3dd..5ba540c0 100644 --- a/internal/firecracker/template.go +++ b/internal/firecracker/template.go @@ -673,36 +673,74 @@ func (tm *TemplateManager) CreateTemplate(id string, cfg VMConfig, initCommands } // normalizeTemplateArtifacts hands every file under the canonical template -// tree back to the daemon's own uid/gid with world-readable modes (files -// 0o644, dirs 0o755) after the jailed build VM is done with it. +// tree back to the daemon's own uid and the shared kvm group with +// group-readable modes (files 0o640, dirs 0o750) after the jailed build VM is +// done with it. // -// Why this exists (#583): in jailer mode the build hardlinks the canonical -// rootfs into the per-VM chroot and chownIntoJail flips the SHARED inode to -// the jailed uid; it must, because the deprivileged build VM writes the -// rootfs. The snapshot mem and vmstate are likewise created inside the jail -// and hardlinked out still owned by the jailed uid. Left that way, the husk -// VMM (uid 0 with ALL capabilities dropped, so no CAP_DAC_OVERRIDE) fails -// EACCES opening the rootfs O_RDWR at /snapshot/load and the warm pool -// crashloops. The bug was latent while the data dir and the chroot base sat -// on different filesystems: prepareChroot's EXDEV fallback copied instead of -// hardlinking, so the chown hit the jail-private copy. Consolidating both -// onto one filesystem made the hardlink succeed and unmasked it. +// Why this exists (#583, #597): in jailer mode the build hardlinks the +// canonical rootfs into the per-VM chroot and chownIntoJail flips the SHARED +// inode to the jailed uid (JailerBuildUID); it must, because the deprivileged +// build VM writes the rootfs. The snapshot mem and vmstate are likewise created +// inside the jail and hardlinked out still owned by the jailed uid. Left that +// way, the husk VMM (uid 0 with ALL capabilities dropped, so no +// CAP_DAC_OVERRIDE) fails EACCES opening the rootfs O_RDWR at /snapshot/load +// and the warm pool crashloops. The bug was latent while the data dir and the +// chroot base sat on different filesystems: prepareChroot's EXDEV fallback +// copied instead of hardlinking, so the chown hit the jail-private copy. +// Consolidating both onto one filesystem made the hardlink succeed and +// unmasked it. // -// Chowning to the daemon's own euid keeps this a no-op in non-jailed and -// non-root paths (mock engine, unit tests) while restoring root ownership on -// a production forkd. +// The owner is the daemon's own euid (root:0 on a production forkd), which +// keeps the current uid-0 husk the file OWNER so its O_RDWR snapshot load still +// works. The GROUP is set to SharedKVMGID and the mode made group-readable so a +// FUTURE non-root husk (issue #585) that carries SharedKVMGID as a supplemental +// group can still read the shared template (its per-activation rootfs reflink +// clone reads rootfs.ext4; the load reads mem and vmstate) without owning the +// files. The artifacts stay NOT world-writable. This requires the ability to +// chgrp to SharedKVMGID, i.e. it runs on a production root forkd; the mock +// engine never calls it. func normalizeTemplateArtifacts(root string) error { - uid, gid := os.Geteuid(), os.Getegid() + uid := os.Geteuid() + // The group is the shared kvm gid so a future non-root husk (issue #585) + // carrying it as a supplemental group reads the template through the group + // class. Only a builder able to chgrp to a group it does not belong to (a + // root forkd) can set it. A non-root builder (standalone sandbox-server, + // self-host) both builds and restores the template as the SAME uid, so it + // needs no shared-group contract and cannot set it. Attempt the shared gid + // and fall back to the builder's own gid on the first permission error; the + // reuse invariant (internal/fork.checkTemplateArtifactInvariants) accepts + // either group so the two stay in lockstep. + gid := SharedKVMGID + fellBack := false return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { return err } - mode := os.FileMode(0o644) + // Never follow symlinks: os.Chown and os.Chmod dereference them, so a + // symlink planted in the build output could redirect this ownership or + // mode change onto a target outside the template tree. Template + // artifacts are regular files and directories; skip anything else, and + // use Lchown so the link inode itself is retargeted, never its target. + if d.Type()&os.ModeSymlink != 0 { + return nil + } + mode := os.FileMode(0o640) if d.IsDir() { - mode = 0o755 + mode = 0o750 } - if err := os.Chown(path, uid, gid); err != nil { - return fmt.Errorf("chown %s: %w", path, err) + if err := os.Lchown(path, uid, gid); err != nil { + if !fellBack && errors.Is(err, os.ErrPermission) { + // Not privileged enough to hand the artifacts to the shared kvm + // group: the non-root builder path. Keep the builder's own gid, + // which it can always set, for this and every remaining entry. + gid = os.Getegid() + fellBack = true + if err := os.Lchown(path, uid, gid); err != nil { + return fmt.Errorf("chown %s: %w", path, err) + } + } else { + return fmt.Errorf("chown %s: %w", path, err) + } } if err := os.Chmod(path, mode); err != nil { return fmt.Errorf("chmod %s: %w", path, err) diff --git a/internal/firecracker/template_owner_test.go b/internal/firecracker/template_owner_test.go index 36c74680..b25caace 100644 --- a/internal/firecracker/template_owner_test.go +++ b/internal/firecracker/template_owner_test.go @@ -1,7 +1,9 @@ package firecracker import ( + "fmt" "os" + "os/exec" "path/filepath" "syscall" "testing" @@ -9,15 +11,24 @@ import ( // The jailed build VM hardlinks the canonical template rootfs into its chroot // and chownIntoJail flips the SHARED inode to the per-VM jailed uid (it must: -// the deprivileged build VM writes the rootfs during the build). Left that -// way, the husk VMM (uid 0 with ALL capabilities dropped, so no -// CAP_DAC_OVERRIDE) fails EACCES opening the rootfs O_RDWR at /snapshot/load -// (#583). normalizeTemplateArtifacts is the correct-by-construction repair at -// the end of CreateTemplate: every canonical template artifact is handed back -// to the daemon's own uid with world-readable modes before the template can -// be registered or digest-recorded. - -func TestNormalizeTemplateArtifactsResetsModes(t *testing.T) { +// the deprivileged build VM writes the rootfs during the build). Left that way, +// a husk VMM that is neither the jailer uid nor privileged fails EACCES opening +// the artifacts (#583, #597). normalizeTemplateArtifacts is the +// correct-by-construction repair at the end of CreateTemplate: every canonical +// template artifact is handed back to the daemon's own uid and the shared kvm +// group with group-readable modes before the template can be registered or +// digest-recorded, so the current uid-0 husk reads them as owner and a future +// non-root husk (issue #585) in the shared kvm group reads them through the +// group class. +// +// Setting the group to SharedKVMGID requires the privilege to chgrp to a +// foreign gid, so the normalize tests run only as root (the KVM e2e runner); +// everywhere else they skip. In production normalize runs on a root forkd. + +func TestNormalizeTemplateArtifactsSetsGroupReadableContract(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root to chgrp template artifacts to the shared kvm gid") + } root := t.TempDir() snapDir := filepath.Join(root, "snapshot") if err := os.MkdirAll(snapDir, 0o700); err != nil { @@ -43,12 +54,12 @@ func TestNormalizeTemplateArtifactsResetsModes(t *testing.T) { if err != nil { t.Fatal(err) } - if got := info.Mode().Perm(); got != 0o644 { - t.Errorf("%s mode = %o, want 0644", f, got) + if got := info.Mode().Perm(); got != 0o640 { + t.Errorf("%s mode = %o, want 0640 (group-readable, not world-writable)", f, got) } st := info.Sys().(*syscall.Stat_t) - if int(st.Uid) != os.Geteuid() || int(st.Gid) != os.Getegid() { - t.Errorf("%s owned %d:%d, want %d:%d", f, st.Uid, st.Gid, os.Geteuid(), os.Getegid()) + if int(st.Uid) != os.Geteuid() || int(st.Gid) != SharedKVMGID { + t.Errorf("%s owned %d:%d, want %d:%d (root:SharedKVMGID)", f, st.Uid, st.Gid, os.Geteuid(), SharedKVMGID) } } for _, d := range []string{root, snapDir} { @@ -56,16 +67,20 @@ func TestNormalizeTemplateArtifactsResetsModes(t *testing.T) { if err != nil { t.Fatal(err) } - if got := info.Mode().Perm(); got != 0o755 { - t.Errorf("%s mode = %o, want 0755", d, got) + if got := info.Mode().Perm(); got != 0o750 { + t.Errorf("%s mode = %o, want 0750", d, got) + } + st := info.Sys().(*syscall.Stat_t) + if int(st.Gid) != SharedKVMGID { + t.Errorf("%s gid = %d, want %d (SharedKVMGID)", d, st.Gid, SharedKVMGID) } } } -// Root-gated regression for the exact production failure: a hardlink of the -// canonical rootfs chowned to a jailed uid flips the shared inode, and -// normalization restores it. Requires CAP_CHOWN, so it runs only as root -// (the KVM e2e runner); everywhere else it skips. +// TestNormalizeTemplateArtifactsRestoresJailFlippedOwner is the root-gated +// regression for the exact production failure (#597): a hardlink of the +// canonical rootfs chowned to the jailer build uid flips the shared inode, and +// normalization restores it to the root:SharedKVMGID group-readable contract. func TestNormalizeTemplateArtifactsRestoresJailFlippedOwner(t *testing.T) { if os.Geteuid() != 0 { t.Skip("requires root to chown to a foreign uid") @@ -81,11 +96,11 @@ func TestNormalizeTemplateArtifactsRestoresJailFlippedOwner(t *testing.T) { t.Fatal(err) } // What chownIntoJail does to the build VM's chroot hardlink. - if err := os.Chown(link, 64000, 64000); err != nil { + if err := os.Chown(link, JailerBuildUID, JailerBuildUID); err != nil { t.Fatal(err) } st := statT(t, rootfs) - if st.Uid != 64000 { + if int(st.Uid) != JailerBuildUID { t.Fatalf("precondition: shared inode not flipped (uid=%d)", st.Uid) } @@ -93,8 +108,178 @@ func TestNormalizeTemplateArtifactsRestoresJailFlippedOwner(t *testing.T) { t.Fatalf("normalizeTemplateArtifacts: %v", err) } st = statT(t, rootfs) - if int(st.Uid) != 0 || int(st.Gid) != 0 { - t.Errorf("rootfs owned %d:%d after normalize, want 0:0", st.Uid, st.Gid) + if int(st.Uid) != 0 || int(st.Gid) != SharedKVMGID { + t.Errorf("rootfs owned %d:%d after normalize, want 0:%d (root:SharedKVMGID)", st.Uid, st.Gid, SharedKVMGID) + } + if got := os.FileMode(st.Mode).Perm(); got != 0o640 { + t.Errorf("rootfs mode = %o after normalize, want 0640", got) + } +} + +// TestNormalizeTemplateArtifactsRestorableByNonRootGroupMember is the key +// proof: after normalize, a NON-ROOT process that is in the shared kvm group +// can open (read) the template artifacts, while a process outside the group +// cannot. This is the read leg every husk restore depends on (issue #597, #585: +// the per-activation rootfs reflink clone reads rootfs.ext4; the snapshot load +// reads mem and vmstate). Dropping privileges in-process is unreliable on Go's +// multi-threaded runtime, so the read is exercised by a child process with +// dropped credentials (uid nobody, primary gid + supplemental group set to the +// shared kvm gid), which is exactly how the husk pod carries the group. +func TestNormalizeTemplateArtifactsRestorableByNonRootGroupMember(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("requires root to normalize to the shared kvm gid and drop privileges") + } + catBin, err := exec.LookPath("cat") + if err != nil { + t.Skipf("cat not found, cannot exercise a dropped-privilege read: %v", err) + } + + root := t.TempDir() + snapDir := filepath.Join(root, "snapshot") + if err := os.MkdirAll(snapDir, 0o700); err != nil { + t.Fatal(err) + } + files := []string{ + filepath.Join(root, "rootfs.ext4"), + filepath.Join(snapDir, "mem"), + filepath.Join(snapDir, "vmstate"), + } + for _, f := range files { + if err := os.WriteFile(f, []byte("payload"), 0o600); err != nil { + t.Fatal(err) + } + } + if err := normalizeTemplateArtifacts(root); err != nil { + t.Fatalf("normalizeTemplateArtifacts: %v", err) + } + // t.TempDir() and its parents must be traversable by the group member, or + // the group read is refused before it reaches the artifact. Widen the search + // path to o+x (no read), which does not affect the artifacts' own 0o640. + // These ancestors can sit outside t.TempDir() (up to /tmp), so capture and + // restore each original mode instead of leaking the widened bit. + for d := root; d != "/" && d != "."; d = filepath.Dir(d) { + info, statErr := os.Stat(d) + if statErr != nil { + break + } + orig := info.Mode().Perm() + dir := d + if err := os.Chmod(dir, orig|0o001); err != nil { + t.Fatalf("widen traverse on %s: %v", dir, err) + } + t.Cleanup(func() { _ = os.Chmod(dir, orig) }) + } + + const nobodyUID = 65534 + + // readAs opens each artifact via cat under the given dropped credentials and + // returns the first error (nil means every read succeeded). + readAs := func(uid, gid uint32, groups []uint32) error { + for _, f := range files { + cmd := exec.Command(catBin, f) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uid, Gid: gid, Groups: groups}, + } + if out, runErr := cmd.CombinedOutput(); runErr != nil { + return fmt.Errorf("%s: %s: %w", f, out, runErr) + } + } + return nil + } + + // In the shared kvm group: every artifact reads without EACCES. + if err := readAs(nobodyUID, SharedKVMGID, []uint32{SharedKVMGID}); err != nil { + t.Fatalf("a non-root member of the shared kvm group must read the normalized template artifacts, got: %v", err) + } + + // Not in the group (a different gid, no matching supplemental group): the + // "other" class has no read bit on a 0o640 file, so the read is refused. + // This proves the group is load-bearing, not that the files are world-read. + const otherGID = 65533 + if err := readAs(nobodyUID, otherGID, []uint32{otherGID}); err == nil { + t.Fatal("a non-root process outside the shared kvm group must NOT be able to read the 0o640 template artifacts") + } +} + +// TestNormalizeTemplateArtifactsFallsBackForNonRootBuilder proves the non-root +// builder path (standalone sandbox-server, self-host): a builder that cannot +// chgrp the artifacts to the shared kvm gid must still succeed, handing them to +// its OWN uid and gid with the group-readable contract, because it restores as +// the same uid it built. This runs in the ordinary non-root test environment +// and regresses the tmpl-smoke failure where normalize aborted with EPERM +// chgrping to the shared kvm gid. +func TestNormalizeTemplateArtifactsFallsBackForNonRootBuilder(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("this exercises the non-root fallback; as root the chgrp to the shared kvm gid succeeds") + } + root := t.TempDir() + snapDir := filepath.Join(root, "snapshot") + if err := os.MkdirAll(snapDir, 0o700); err != nil { + t.Fatal(err) + } + files := []string{ + filepath.Join(root, "rootfs.ext4"), + filepath.Join(snapDir, "mem"), + filepath.Join(snapDir, "vmstate"), + } + for _, f := range files { + if err := os.WriteFile(f, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + } + + if err := normalizeTemplateArtifacts(root); err != nil { + t.Fatalf("normalizeTemplateArtifacts must succeed for a non-root builder, got: %v", err) + } + + wantUID, wantGID := os.Geteuid(), os.Getegid() + for _, f := range files { + st := statT(t, f) + if int(st.Uid) != wantUID || int(st.Gid) != wantGID { + t.Errorf("%s owned %d:%d, want %d:%d (the builder's own uid:gid)", f, st.Uid, st.Gid, wantUID, wantGID) + } + if got := os.FileMode(st.Mode).Perm(); got != 0o640 { + t.Errorf("%s mode = %o, want 0640", f, got) + } + } + for _, d := range []string{root, snapDir} { + st := statT(t, d) + if got := os.FileMode(st.Mode).Perm(); got != 0o750 { + t.Errorf("%s mode = %o, want 0750", d, got) + } + } +} + +// TestNormalizeTemplateArtifactsSkipsSymlinks proves normalize never follows a +// symlink in the build tree. os.Chown and os.Chmod dereference symlinks, so a +// link planted in the build output could redirect a privileged ownership or +// mode change onto a target outside the template directory; normalize must +// leave that target untouched (addresses the CodeRabbit traversal finding). +func TestNormalizeTemplateArtifactsSkipsSymlinks(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "rootfs.ext4"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + // A target OUTSIDE the template tree, and a symlink to it planted inside. + outside := filepath.Join(t.TempDir(), "victim") + if err := os.WriteFile(outside, []byte("y"), 0o600); err != nil { + t.Fatal(err) + } + before := statT(t, outside) + if err := os.Symlink(outside, filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + + if err := normalizeTemplateArtifacts(root); err != nil { + t.Fatalf("normalizeTemplateArtifacts: %v", err) + } + + after := statT(t, outside) + if before.Uid != after.Uid || before.Gid != after.Gid || + os.FileMode(before.Mode).Perm() != os.FileMode(after.Mode).Perm() { + t.Errorf("symlink target changed %d:%d %o -> %d:%d %o; normalize followed a symlink out of the template tree", + before.Uid, before.Gid, os.FileMode(before.Mode).Perm(), + after.Uid, after.Gid, os.FileMode(after.Mode).Perm()) } } diff --git a/internal/fork/template_invariants.go b/internal/fork/template_invariants.go index 5dbacdfd..5627424e 100644 --- a/internal/fork/template_invariants.go +++ b/internal/fork/template_invariants.go @@ -6,6 +6,8 @@ import ( "path/filepath" "sort" "syscall" + + "mitos.run/mitos/internal/firecracker" ) // Reuse-or-rebuild gate (issue #584): a template discovered on disk (typically @@ -15,47 +17,83 @@ import ( // ownership invariants enforced here (proves the jailed build did not leave // the artifacts in a state the husk VMM cannot read). // -// The production trigger for the invariant check is incident #583: the -// jailed build flips template artifact ownership to uid 64000 (the jailer's -// unprivileged build uid), which a husk VMM running as a different euid -// cannot open. A companion change (#587) normalizes ownership back to the -// running process's euid at the end of a successful build -// (normalizeTemplateArtifacts); this function is the READ-side gate that -// refuses to reuse a template that was never normalized, or whose ownership -// regressed after the fact, rather than trusting an on-disk template blindly. +// The production trigger for the invariant check is incident #583/#597: the +// jailed build flips template artifact ownership to the jailer's unprivileged +// build uid (firecracker.JailerBuildUID), which a husk VMM running as a +// different uid cannot open. A companion change (#587, extended by #597) +// normalizes ownership at the end of a successful build +// (normalizeTemplateArtifacts) to root:SharedKVMGID with a group-readable mode; +// this function is the READ-side gate that refuses to reuse a template that was +// never normalized, or whose ownership regressed after the fact, rather than +// trusting an on-disk template blindly. // checkTemplateArtifactInvariants verifies that every snapshot artifact of // template id under dataDir (rootfs.ext4 when present, snapshot/mem, -// snapshot/vmstate) is owned by the calling process's effective uid and -// carries mode 0o644. It returns a descriptive error naming the offending -// file, its actual owner, and the expected owner on the first invariant that +// snapshot/vmstate) matches the normalized ownership contract: owned by the +// calling process's effective uid, group-owned by the shared kvm gid +// (firecracker.SharedKVMGID), and carrying the group-readable file mode 0o640. +// This is the read side of the contract normalizeTemplateArtifacts writes at +// build time: root:SharedKVMGID, group-readable, not world-writable, so the +// current uid-0 husk reads as owner and a future non-root husk (issue #585) in +// the SharedKVMGID group reads through the group class. It returns a +// descriptive error naming the offending file on the first invariant that // fails; artifacts are checked in a stable (sorted) order so the error is // deterministic across runs. func checkTemplateArtifactInvariants(dataDir, id string) error { wantUID := os.Geteuid() - files := templateSnapshotFiles(dataDir, id) - - names := make([]string, 0, len(files)) - for name := range files { - names = append(names, name) - } - sort.Strings(names) - for _, name := range names { - path := files[name] + // checkOwnership enforces the shared uid/gid contract on one entry. kind is + // "artifact" or "dir" for the message; wantMode is 0o640 for files and + // 0o750 for directories. + checkOwnership := func(kind, path string, wantMode os.FileMode) error { info, err := os.Stat(path) if err != nil { - return fmt.Errorf("stat template %s artifact %s: %w", id, path, err) + return fmt.Errorf("stat template %s %s %s: %w", id, kind, path, err) } st, ok := info.Sys().(*syscall.Stat_t) if !ok { - return fmt.Errorf("template %s artifact %s: cannot determine owner on this platform", id, path) + return fmt.Errorf("template %s %s %s: cannot determine owner on this platform", id, kind, path) } if gotUID := int(st.Uid); gotUID != wantUID { - return fmt.Errorf("template %s artifact %s is owned by uid %d, expected uid %d (this process's euid); the jailed build likely flipped ownership (issue #583), so the template is unusable until rebuilt or its ownership is fixed", id, path, gotUID, wantUID) + return fmt.Errorf("template %s %s %s is owned by uid %d, expected uid %d (this process's euid); the jailed build likely flipped ownership (issues #583, #597), so the template is unusable until rebuilt or its ownership is fixed", id, kind, path, gotUID, wantUID) + } + // A root builder tags artifacts with the shared kvm gid so a separate + // husk uid reads them through the group class. A non-root builder + // (standalone sandbox-server, self-host) cannot chgrp to that group and + // does not need to: it restores as the same uid that built, so its own + // gid is correct. Accept either, mirroring normalizeTemplateArtifacts' + // fallback, so a non-root deployment reuses its templates instead of + // rebuilding every one. + if gotGID := int(st.Gid); gotGID != firecracker.SharedKVMGID && gotGID != os.Getegid() { + return fmt.Errorf("template %s %s %s is group-owned by gid %d, expected gid %d (the shared kvm group a husk reads the template through, issues #585, #597) or this process's gid %d; the template is unusable until rebuilt or its group is fixed", id, kind, path, gotGID, firecracker.SharedKVMGID, os.Getegid()) + } + if mode := info.Mode().Perm(); mode != wantMode { + return fmt.Errorf("template %s %s %s has mode %#o, expected mode %#o; the template is unusable until rebuilt or its mode is fixed", id, kind, path, mode, wantMode) + } + return nil + } + + // Check the containing directories first: normalizeTemplateArtifacts sets + // them to 0o750, and a husk cannot traverse to the files (EACCES) if the + // template or snapshot dir has lost group execute, even when the files + // themselves are compliant. + dir := templateDir(dataDir, id) + for _, d := range []string{dir, filepath.Join(dir, "snapshot")} { + if err := checkOwnership("dir", d, 0o750); err != nil { + return err } - if mode := info.Mode().Perm(); mode != 0o644 { - return fmt.Errorf("template %s artifact %s has mode %#o, expected mode 0o644; the template is unusable until rebuilt or its mode is fixed", id, path, mode) + } + + files := templateSnapshotFiles(dataDir, id) + names := make([]string, 0, len(files)) + for name := range files { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + if err := checkOwnership("artifact", files[name], 0o640); err != nil { + return err } } return nil diff --git a/internal/fork/template_reuse_engine_test.go b/internal/fork/template_reuse_engine_test.go index 126b8eeb..7f2dd654 100644 --- a/internal/fork/template_reuse_engine_test.go +++ b/internal/fork/template_reuse_engine_test.go @@ -59,9 +59,16 @@ func writeRebuiltTemplate(t *testing.T, dataDir, id string) { // case: a healthy on-disk template (digest verifies, ownership invariants // pass) must be reused, so runTemplateBuild must never run. func TestCreateTemplateEngineReusesHealthyTemplate(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("a reusable template must be group-owned by the shared kvm gid, which requires root to set up") + } e := newTestEngine(t) id := "py" seedHealthyTemplate(t, e, id) + // The reuse gate now also requires the normalized root:SharedKVMGID + // group-readable ownership contract (issue #597), which the seed cannot set + // non-root; apply it here so the gate sees a healthy template. + makeTemplateArtifactsCompliant(t, e.dataDir, id) built := false e.runTemplateBuild = func(string, firecracker.VMConfig, []string, *firecracker.WorkloadSpec, bool) error { diff --git a/internal/fork/template_reuse_test.go b/internal/fork/template_reuse_test.go index 404d9971..6eca04c6 100644 --- a/internal/fork/template_reuse_test.go +++ b/internal/fork/template_reuse_test.go @@ -7,27 +7,92 @@ import ( "testing" "mitos.run/mitos/internal/cas" + "mitos.run/mitos/internal/firecracker" ) -// TestCheckTemplateArtifactInvariantsOwnedByEuid is the happy path: a -// template fixture written by this test process is owned by the process's -// own euid and carries mode 0o644 on every artifact (mustWrite in -// verify_test.go already writes 0o644), so the invariant check must pass. -func TestCheckTemplateArtifactInvariantsOwnedByEuid(t *testing.T) { +// makeTemplateArtifactsCompliant rewrites the on-disk template artifacts to the +// normalized ownership contract (this process's euid, the shared kvm group, and +// the group-readable mode 0o640) so a root-gated test can exercise the +// checkTemplateArtifactInvariants happy path. It requires the privilege to +// chgrp to the shared kvm gid, so every caller must root-gate. +func makeTemplateArtifactsCompliant(t *testing.T, dataDir, id string) { + t.Helper() + // The containing directories must be group-traversable (0o750, shared kvm + // gid) too, mirroring normalizeTemplateArtifacts, or the reuse invariant + // rejects on the dir before it ever reaches the files. + dir := filepath.Join(dataDir, "templates", id) + for _, d := range []string{dir, filepath.Join(dir, "snapshot")} { + if err := os.Chown(d, os.Geteuid(), firecracker.SharedKVMGID); err != nil { + t.Fatalf("chown %s: %v", d, err) + } + if err := os.Chmod(d, 0o750); err != nil { + t.Fatalf("chmod %s: %v", d, err) + } + } + for _, p := range templateSnapshotFiles(dataDir, id) { + if err := os.Chown(p, os.Geteuid(), firecracker.SharedKVMGID); err != nil { + t.Fatalf("chown %s: %v", p, err) + } + if err := os.Chmod(p, 0o640); err != nil { + t.Fatalf("chmod %s: %v", p, err) + } + } +} + +// TestCheckTemplateArtifactInvariantsRejectsUnnormalized runs everywhere +// (non-root CI included): a freshly written fixture is mode 0o644, i.e. it was +// NEVER normalized to the group-readable 0o640 contract. The invariant gate +// must refuse it and name the offending path, proving the reuse gate is active +// without needing root to construct a foreign-group fixture. The fixture is +// owned by the test process's own gid, which the gate accepts for a non-root +// builder (a root builder tags the shared kvm gid instead), so the mode is the +// difference the gate catches here. +func TestCheckTemplateArtifactInvariantsRejectsUnnormalized(t *testing.T) { id := "py" dataDir := writeFakeTemplate(t, id) + // The gate checks artifacts in a stable sorted order (mem, rootfs, vmstate), + // so it names the first non-compliant one; assert it names an artifact under + // this template's dir and cites the group-readable mode mismatch. + templatePath := filepath.Join(dataDir, "templates", id) + + err := checkTemplateArtifactInvariants(dataDir, id) + if err == nil { + t.Fatal("expected checkTemplateArtifactInvariants to reject an un-normalized template") + } + if !strings.Contains(err.Error(), templatePath) { + t.Fatalf("error %q does not name an artifact under %q", err.Error(), templatePath) + } + if !strings.Contains(err.Error(), "mode") { + t.Fatalf("error %q should cite the group-readable mode mismatch", err.Error()) + } +} + +// TestCheckTemplateArtifactInvariantsCompliant is the happy path: a template +// normalized to euid:SharedKVMGID at mode 0o640 clears the invariant gate. +// Requires root to chgrp to the shared kvm gid, so it is skipped otherwise. +func TestCheckTemplateArtifactInvariantsCompliant(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("chgrp to the shared kvm gid requires root") + } + id := "py" + dataDir := writeFakeTemplate(t, id) + makeTemplateArtifactsCompliant(t, dataDir, id) if err := checkTemplateArtifactInvariants(dataDir, id); err != nil { t.Fatalf("checkTemplateArtifactInvariants: unexpected error %v", err) } } -// TestCheckTemplateArtifactInvariantsWrongMode writes the rootfs with mode -// 0o600 instead of the required 0o644 and asserts the error names the -// offending path. +// TestCheckTemplateArtifactInvariantsWrongMode starts from a compliant template +// and drops the rootfs mode to 0o600 (no group read), then asserts the error +// names the offending path. Root-gated so the compliant base can be built. func TestCheckTemplateArtifactInvariantsWrongMode(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("chgrp to the shared kvm gid requires root") + } id := "py" dataDir := writeFakeTemplate(t, id) + makeTemplateArtifactsCompliant(t, dataDir, id) rootfsPath := filepath.Join(dataDir, "templates", id, "rootfs.ext4") if err := os.Chmod(rootfsPath, 0o600); err != nil { t.Fatalf("chmod rootfs: %v", err) @@ -42,18 +107,18 @@ func TestCheckTemplateArtifactInvariantsWrongMode(t *testing.T) { } } -// TestCheckTemplateArtifactInvariantsForeignOwner chowns the rootfs to a -// foreign uid (64000, the jailer's build uid per issue #583) and asserts the -// invariant check rejects it. Requires root to chown, so it is skipped -// otherwise. +// TestCheckTemplateArtifactInvariantsForeignOwner chowns the rootfs to the +// jailer's build uid (issue #583/#597) and asserts the invariant check rejects +// it. Requires root to chown, so it is skipped otherwise. func TestCheckTemplateArtifactInvariantsForeignOwner(t *testing.T) { if os.Geteuid() != 0 { t.Skip("chown to a foreign uid requires root") } id := "py" dataDir := writeFakeTemplate(t, id) + makeTemplateArtifactsCompliant(t, dataDir, id) rootfsPath := filepath.Join(dataDir, "templates", id, "rootfs.ext4") - if err := os.Chown(rootfsPath, 64000, 64000); err != nil { + if err := os.Chown(rootfsPath, firecracker.JailerBuildUID, firecracker.SharedKVMGID); err != nil { t.Fatalf("chown rootfs: %v", err) } @@ -66,6 +131,32 @@ func TestCheckTemplateArtifactInvariantsForeignOwner(t *testing.T) { } } +// TestCheckTemplateArtifactInvariantsWrongGroup chgrps the rootfs to a group +// other than SharedKVMGID and asserts the invariant check rejects it: a husk +// that is not in the artifact's group cannot read it, so the template is not +// reusable. Requires root to chgrp, so it is skipped otherwise. +func TestCheckTemplateArtifactInvariantsWrongGroup(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("chgrp to a foreign gid requires root") + } + id := "py" + dataDir := writeFakeTemplate(t, id) + makeTemplateArtifactsCompliant(t, dataDir, id) + rootfsPath := filepath.Join(dataDir, "templates", id, "rootfs.ext4") + const otherGID = firecracker.SharedKVMGID + 1 + if err := os.Chown(rootfsPath, os.Geteuid(), otherGID); err != nil { + t.Fatalf("chgrp rootfs: %v", err) + } + + err := checkTemplateArtifactInvariants(dataDir, id) + if err == nil { + t.Fatal("expected checkTemplateArtifactInvariants to fail on wrong group") + } + if !strings.Contains(err.Error(), rootfsPath) { + t.Fatalf("error %q does not name the offending path %q", err.Error(), rootfsPath) + } +} + // TestShouldReuseTemplateNoOnDiskTemplate exercises the decision seam used by // the CreateTemplate reuse-or-rebuild gate (#584) when no template exists on // disk yet: verify must never be called, and the answer is "do not reuse" @@ -90,10 +181,15 @@ func TestShouldReuseTemplateNoOnDiskTemplate(t *testing.T) { // TestCreateTemplateReusesHealthyOnDisk exercises the reuse seam directly: a // healthy on-disk template (recorded digest verifies, artifacts pass the -// ownership invariants) must be reused. +// ownership invariants) must be reused. Root-gated because the compliant +// artifacts must be group-owned by the shared kvm gid. func TestCreateTemplateReusesHealthyOnDisk(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("chgrp to the shared kvm gid requires root") + } id := "py" dataDir := writeFakeTemplate(t, id) + makeTemplateArtifactsCompliant(t, dataDir, id) store := newTestStore(t, dataDir) if _, err := recordTemplateDigest(store, dataDir, id, cas.Metadata{}); err != nil { t.Fatalf("recordTemplateDigest: %v", err) @@ -116,9 +212,10 @@ func TestCreateTemplateReusesHealthyOnDisk(t *testing.T) { } // TestCreateTemplateRebuildsBrokenOnDisk exercises the reuse seam when the -// on-disk template fails digest verification (tampered content): the gate -// must refuse reuse and surface the verification error so the caller can -// delete and rebuild. +// on-disk template fails digest verification (tampered content): the gate must +// refuse reuse and surface the verification error so the caller can delete and +// rebuild. Digest verification runs before the ownership invariants, so this +// runs non-root. func TestCreateTemplateRebuildsBrokenOnDisk(t *testing.T) { id := "py" dataDir := writeFakeTemplate(t, id) @@ -148,10 +245,15 @@ func TestCreateTemplateRebuildsBrokenOnDisk(t *testing.T) { // TestCreateTemplateRebuildsBrokenInvariants exercises the reuse seam when // digest verification succeeds but the artifact ownership invariants fail -// (issue #583): the gate must still refuse reuse. +// (issue #583/#597): the gate must still refuse reuse. Root-gated so the +// compliant-then-broken base can be built. func TestCreateTemplateRebuildsBrokenInvariants(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("chgrp to the shared kvm gid requires root") + } id := "py" dataDir := writeFakeTemplate(t, id) + makeTemplateArtifactsCompliant(t, dataDir, id) store := newTestStore(t, dataDir) if _, err := recordTemplateDigest(store, dataDir, id, cas.Metadata{}); err != nil { t.Fatalf("recordTemplateDigest: %v", err) diff --git a/internal/husk/stub.go b/internal/husk/stub.go index 0404c1d4..69fbd47e 100644 --- a/internal/husk/stub.go +++ b/internal/husk/stub.go @@ -928,6 +928,23 @@ func (s *Stub) Activate(ctx context.Context, req ActivateRequest) (ActivateResul } } + // Mandatory per-activation rootfs CoW (issue #597): when this husk was + // configured with a shared template rootfs, the ONLY correct restore rebinds + // the baked rootfs drive to THIS activation's own reflink clone (prepared + // during the dormant window); the resumed guest must never write, and the + // pool must never silently in-place restore, the shared template. If the + // clone is missing (the CoW dir was not configured, or the Prepare clone did + // not run) we FAIL CLOSED here rather than fall back to opening the shared + // template: an in-place restore both corrupts concurrent activations of one + // template and, once the template artifacts are group-readable to a non-owner + // husk (issue #585), is the exact O_RDWR-open EACCES this change removes. + // rootfsTemplatePath empty means no shared template is in play (the raw + // control-socket and unit-test paths), which keeps the prior behavior. + if s.rootfsTemplatePath != "" && s.rootfsClonePath == "" { + werr := fmt.Errorf("husk: per-activation rootfs CoW required for template %s but no clone was prepared; --rootfs-cow-dir must be configured, refusing in-place restore of the shared template", s.rootfsTemplatePath) + return ActivateResult{OK: false, Error: werr.Error()}, werr + } + // Load the snapshot PAUSED (resume=false). The rootfs drive rebind below MUST // happen before the guest runs, and PATCH /drives on the ROOT device of an // already-RESUMED VM both leaves a write window (any writeback between resume diff --git a/internal/husk/stub_test.go b/internal/husk/stub_test.go index dd6f2d6c..d15cde96 100644 --- a/internal/husk/stub_test.go +++ b/internal/husk/stub_test.go @@ -900,6 +900,55 @@ func TestActivateNoRebindWhenNoClone(t *testing.T) { } } +// TestActivateFailsClosedWhenTemplateWithoutClone is the mandatory-CoW guard +// (issue #597): a husk configured with a SHARED template rootfs but no +// per-activation clone (the CoW dir was not configured, so Prepare took no +// clone) must REFUSE to activate rather than fall back to loading the shared +// template in place O_RDWR. The snapshot must never load, so Firecracker never +// opens the shared template. +func TestActivateFailsClosedWhenTemplateWithoutClone(t *testing.T) { + dir := t.TempDir() + tmplRootfs := filepath.Join(dir, "rootfs.ext4") + if err := os.WriteFile(tmplRootfs, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + vm := &fakeVMM{} + // A shared template rootfs is configured, but no CoW dir: Prepare skips the + // clone (rootfsClonePath stays empty) and stays dormant. + s := New(firecracker.VMConfig{ID: "husk-test"}, Options{ + Start: func(cfg firecracker.VMConfig) (vmm, error) { return vm, nil }, + Ready: readyOK, + Notify: (&fakeNotifier{}).notify, + Verify: verifyOK, + RootfsTemplatePath: tmplRootfs, + // RootfsCoWDir intentionally empty (missing CoW config). + }) + if err := s.Prepare(context.Background()); err != nil { + t.Fatalf("Prepare: %v", err) + } + + res, err := s.Activate(context.Background(), ActivateRequest{SnapshotDir: filepath.Join(dir, "snap")}) + if err == nil { + t.Fatal("expected Activate to fail closed when a template rootfs has no CoW clone") + } + if res.OK { + t.Fatal("fail closed: result must not be OK without a per-activation clone") + } + if !strings.Contains(res.Error, "CoW") && !strings.Contains(res.Error, "clone") { + t.Errorf("error %q should explain the missing per-activation clone", res.Error) + } + // The load MUST NOT have run: Firecracker never opens the shared template. + if vm.loadCalls != 0 { + t.Errorf("the shared template must never be loaded in place, got %d loads", vm.loadCalls) + } + if len(vm.patchCalls) != 0 { + t.Errorf("no clone was prepared, so no PatchDrive expected, got %v", vm.patchCalls) + } + if s.State() == StateActive { + t.Error("state must not be active after a fail-closed activate") + } +} + func TestActivateRebindFailureFailsClosed(t *testing.T) { dir := t.TempDir() tmplRootfs := filepath.Join(dir, "rootfs.ext4")